Index: llvm/trunk/include/llvm/Analysis/InlineCost.h =================================================================== --- llvm/trunk/include/llvm/Analysis/InlineCost.h +++ llvm/trunk/include/llvm/Analysis/InlineCost.h @@ -28,16 +28,26 @@ class TargetTransformInfo; namespace InlineConstants { - // Various magic constants used to adjust heuristics. - const int InstrCost = 5; - const int IndirectCallThreshold = 100; - const int CallPenalty = 25; - const int LastCallToStaticBonus = -15000; - const int ColdccPenalty = 2000; - const int NoreturnPenalty = 10000; - /// Do not inline functions which allocate this many bytes on the stack - /// when the caller is recursive. - const unsigned TotalAllocaSizeRecursiveCaller = 1024; +// Various thresholds used by inline cost analysis. +// Use when optsize (-Os) is specified. +const int OptSizeThreshold = 75; + +// Use when minsize (-Oz) is specified. +const int OptMinSizeThreshold = 25; + +// Use when -O3 is specified. +const int OptAggressiveThreshold = 275; + +// Various magic constants used to adjust heuristics. +const int InstrCost = 5; +const int IndirectCallThreshold = 100; +const int CallPenalty = 25; +const int LastCallToStaticBonus = -15000; +const int ColdccPenalty = 2000; +const int NoreturnPenalty = 10000; +/// Do not inline functions which allocate this many bytes on the stack +/// when the caller is recursive. +const unsigned TotalAllocaSizeRecursiveCaller = 1024; } /// \brief Represents the cost of inlining a function. @@ -100,6 +110,52 @@ int getCostDelta() const { return Threshold - getCost(); } }; +/// Thresholds to tune inline cost analysis. The inline cost analysis decides +/// the condition to apply a threshold and applies it. Otherwise, +/// DefaultThreshold is used. If a threshold is Optional, it is applied only +/// when it has a valid value. Typically, users of inline cost analysis +/// obtain an InlineParams object through one of the \c getInlineParams methods +/// and pass it to \c getInlineCost. Some specialized versions of inliner +/// (such as the pre-inliner) might have custom logic to compute \c InlineParams +/// object. + +struct InlineParams { + /// The default threshold to start with for a callee. + int DefaultThreshold; + + /// Threshold to use for callees with inline hint. + int HintThreshold; + + /// Threshold to use for cold callees. + Optional ColdThreshold; + + /// Threshold to use when the caller is optimized for size. + Optional OptSizeThreshold; + + /// Threshold to use when the caller is optimized for minsize. + Optional OptMinSizeThreshold; + + /// Threshold to use when the callsite is considered hot. + int HotCallSiteThreshold; +}; + +/// Generate the parameters to tune the inline cost analysis based only on the +/// commandline options. +InlineParams getInlineParams(); + +/// Generate the parameters to tune the inline cost analysis based on command +/// line options. If -inline-threshold option is not explicitly passed, +/// \p Threshold is used as the default threshold. +InlineParams getInlineParams(int Threshold); + +/// Generate the parameters to tune the inline cost analysis based on command +/// line options. If -inline-threshold option is not explicitly passed, +/// the default threshold is computed from \p OptLevel and \p SizeOptLevel. +/// An \p OptLevel value above 3 is considered an aggressive optimization mode. +/// \p SizeOptLevel of 1 corresponds to the the -Os flag and 2 corresponds to +/// the -Oz flag. +InlineParams getInlineParams(unsigned OptLevel, unsigned SizeOptLevel); + /// \brief Get an InlineCost object representing the cost of inlining this /// callsite. /// @@ -112,7 +168,8 @@ /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. InlineCost -getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, +getInlineCost(CallSite CS, const InlineParams &Params, + TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, ProfileSummaryInfo *PSI); @@ -122,16 +179,11 @@ /// parameter in all other respects. // InlineCost -getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold, +getInlineCost(CallSite CS, Function *Callee, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, ProfileSummaryInfo *PSI); -int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel); - -/// \brief Return the default value of -inline-threshold. -int getDefaultInlineThreshold(); - /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); } Index: llvm/trunk/lib/Analysis/InlineCost.cpp =================================================================== --- llvm/trunk/lib/Analysis/InlineCost.cpp +++ llvm/trunk/lib/Analysis/InlineCost.cpp @@ -40,18 +40,7 @@ STATISTIC(NumCallsAnalyzed, "Number of call sites analyzed"); -// Threshold to use when optsize is specified (and there is no -// -inline-threshold). -const int OptSizeThreshold = 75; - -// Threshold to use when -Oz is specified (and there is no -inline-threshold). -const int OptMinSizeThreshold = 25; - -// Threshold to use when -O[34] is specified (and there is no -// -inline-threshold). -const int OptAggressiveThreshold = 275; - -static cl::opt DefaultInlineThreshold( +static cl::opt InlineThreshold( "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, cl::desc("Control the amount of inlining to perform (default = 225)")); @@ -94,6 +83,9 @@ // easily cacheable. Instead, use the cover function paramHasAttr. CallSite CandidateCS; + // Tunable parameters that control the analysis. + const InlineParams &Params; + int Threshold; int Cost; @@ -210,11 +202,11 @@ public: CallAnalyzer(const TargetTransformInfo &TTI, std::function &GetAssumptionCache, - ProfileSummaryInfo *PSI, Function &Callee, int Threshold, - CallSite CSArg) + ProfileSummaryInfo *PSI, Function &Callee, CallSite CSArg, + const InlineParams &Params) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), PSI(PSI), F(Callee), - CandidateCS(CSArg), Threshold(Threshold), Cost(0), - IsCallerRecursive(false), IsRecursiveCall(false), + CandidateCS(CSArg), Params(Params), Threshold(Params.DefaultThreshold), + Cost(0), IsCallerRecursive(false), IsRecursiveCall(false), ExposesReturnsTwice(false), HasDynamicAlloca(false), ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), HasFrameEscape(false), AllocatedSize(0), NumInstructions(0), @@ -626,18 +618,18 @@ } Function *Caller = CS.getCaller(); - if (DefaultInlineThreshold.getNumOccurrences() > 0) { - // Explicitly specified -inline-threhold overrides the threshold passed to - // CallAnalyzer's constructor. - Threshold = DefaultInlineThreshold; - } else { - // If -inline-threshold is not given, listen to the optsize and minsize - // attributes when they would decrease the threshold. - if (Caller->optForMinSize() && OptMinSizeThreshold < Threshold) - Threshold = OptMinSizeThreshold; - else if (Caller->optForSize() && OptSizeThreshold < Threshold) - Threshold = OptSizeThreshold; - } + + // return min(A, B) if B is valid. + auto MinIfValid = [](int A, Optional B) { + return B ? std::min(A, B.getValue()) : A; + }; + + // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available + // and reduce the threshold if the caller has the necessary attribute. + if (Caller->optForMinSize()) + Threshold = MinIfValid(Threshold, Params.OptMinSizeThreshold); + else if (Caller->optForSize()) + Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); bool HotCallsite = false; uint64_t TotalWeight; @@ -651,21 +643,18 @@ // minimize its size. bool InlineHint = Callee.hasFnAttribute(Attribute::InlineHint) || PSI->isHotFunction(&Callee); - if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) - Threshold = HintThreshold; + if (InlineHint && !Caller->optForMinSize()) + Threshold = std::max(Threshold, Params.HintThreshold); if (HotCallsite && HotCallSiteThreshold > Threshold && !Caller->optForMinSize()) - Threshold = HotCallSiteThreshold; + Threshold = std::max(Threshold, Params.HotCallSiteThreshold); bool ColdCallee = PSI->isColdFunction(&Callee); - // Command line argument for DefaultInlineThreshold will override the default - // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, - // do not use the default cold threshold even if it is smaller. - if ((DefaultInlineThreshold.getNumOccurrences() == 0 || - ColdThreshold.getNumOccurrences() > 0) && - ColdCallee && ColdThreshold < Threshold) - Threshold = ColdThreshold; + // For cold callees, use the ColdThreshold knob if it is available and reduces + // the threshold. + if (ColdCallee) + Threshold = MinIfValid(Threshold, Params.ColdThreshold); // Finally, take the target-specific inlining threshold multiplier into // account. @@ -967,8 +956,9 @@ // during devirtualization and so we want to give it a hefty bonus for // inlining, but cap that bonus in the event that inlining wouldn't pan // out. Pretend to inline the function, with a custom threshold. - CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, - InlineConstants::IndirectCallThreshold, CS); + auto IndirectCallParams = Params; + IndirectCallParams.DefaultThreshold = InlineConstants::IndirectCallThreshold; + CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F, CS, Params); if (CA.analyzeCall(CS)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. @@ -1454,28 +1444,15 @@ } InlineCost llvm::getInlineCost( - CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI, + CallSite CS, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, ProfileSummaryInfo *PSI) { - return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI, + return getInlineCost(CS, CS.getCalledFunction(), Params, CalleeTTI, GetAssumptionCache, PSI); } -int llvm::computeThresholdFromOptLevels(unsigned OptLevel, - unsigned SizeOptLevel) { - if (OptLevel > 2) - return OptAggressiveThreshold; - if (SizeOptLevel == 1) // -Os - return OptSizeThreshold; - if (SizeOptLevel == 2) // -Oz - return OptMinSizeThreshold; - return DefaultInlineThreshold; -} - -int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; } - InlineCost llvm::getInlineCost( - CallSite CS, Function *Callee, int DefaultThreshold, + CallSite CS, Function *Callee, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, ProfileSummaryInfo *PSI) { @@ -1512,7 +1489,7 @@ DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "...\n"); - CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, CS); + CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, CS, Params); bool ShouldInline = CA.analyzeCall(CS); DEBUG(CA.dump()); @@ -1560,3 +1537,67 @@ return true; } + +// APIs to create InlineParams based on command line flags and/or other +// parameters. + +InlineParams llvm::getInlineParams(int Threshold) { + InlineParams Params; + + // This field is the threshold to use for a callee by default. This is + // derived from one or more of: + // * optimization or size-optimization levels, + // * a value passed to createFunctionInliningPass function, or + // * the -inline-threshold flag. + // If the -inline-threshold flag is explicitly specified, that is used + // irrespective of anything else. + if (InlineThreshold.getNumOccurrences() > 0) + Params.DefaultThreshold = InlineThreshold; + else + Params.DefaultThreshold = Threshold; + + // Set the HintThreshold knob from the -inlinehint-threshold. + Params.HintThreshold = HintThreshold; + + // Set the HotCallSiteThreshold knob from the -hot-callsite-threshold. + Params.HotCallSiteThreshold = HotCallSiteThreshold; + + // Set the OptMinSizeThreshold and OptSizeThreshold params only if the + // Set the OptMinSizeThreshold and OptSizeThreshold params only if the + // -inlinehint-threshold commandline option is not explicitly given. If that + // option is present, then its value applies even for callees with size and + // minsize attributes. + // If the -inline-threshold is not specified, set the ColdThreshold from the + // -inlinecold-threshold even if it is not explicitly passed. If + // -inline-threshold is specified, then -inlinecold-threshold needs to be + // explicitly specified to set the ColdThreshold knob + if (InlineThreshold.getNumOccurrences() == 0) { + Params.OptMinSizeThreshold = InlineConstants::OptMinSizeThreshold; + Params.OptSizeThreshold = InlineConstants::OptSizeThreshold; + Params.ColdThreshold = ColdThreshold; + } else if (ColdThreshold.getNumOccurrences() > 0) { + Params.ColdThreshold = ColdThreshold; + } + return Params; +} + +InlineParams llvm::getInlineParams() { + return getInlineParams(InlineThreshold); +} + +// Compute the default threshold for inlining based on the opt level and the +// size opt level. +static int computeThresholdFromOptLevels(unsigned OptLevel, + unsigned SizeOptLevel) { + if (OptLevel > 2) + return InlineConstants::OptAggressiveThreshold; + if (SizeOptLevel == 1) // -Os + return InlineConstants::OptSizeThreshold; + if (SizeOptLevel == 2) // -Oz + return InlineConstants::OptMinSizeThreshold; + return InlineThreshold; +} + +InlineParams llvm::getInlineParams(unsigned OptLevel, unsigned SizeOptLevel) { + return getInlineParams(computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); +} Index: llvm/trunk/lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/InlineSimple.cpp +++ llvm/trunk/lib/Transforms/IPO/InlineSimple.cpp @@ -39,20 +39,15 @@ /// inliner pass and the always inliner pass. The two passes use different cost /// analyses to determine when to inline. class SimpleInliner : public Inliner { - // This field is populated based on one of the following: - // * optimization or size-optimization levels, - // * the --inline-threshold flag, or - // * a user specified value. - int DefaultThreshold; + + InlineParams Params; public: - SimpleInliner() - : Inliner(ID), DefaultThreshold(llvm::getDefaultInlineThreshold()) { + SimpleInliner() : Inliner(ID), Params(llvm::getInlineParams()) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } - explicit SimpleInliner(int Threshold) - : Inliner(ID), DefaultThreshold(Threshold) { + explicit SimpleInliner(InlineParams Params) : Inliner(ID), Params(Params) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } @@ -65,8 +60,7 @@ [&](Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); }; - return llvm::getInlineCost(CS, DefaultThreshold, TTI, GetAssumptionCache, - PSI); + return llvm::getInlineCost(CS, Params, TTI, GetAssumptionCache, PSI); } bool runOnSCC(CallGraphSCC &SCC) override; @@ -74,6 +68,7 @@ private: TargetTransformInfoWrapperPass *TTIWP; + }; } // end anonymous namespace @@ -92,13 +87,12 @@ Pass *llvm::createFunctionInliningPass() { return new SimpleInliner(); } Pass *llvm::createFunctionInliningPass(int Threshold) { - return new SimpleInliner(Threshold); + return new SimpleInliner(llvm::getInlineParams(Threshold)); } Pass *llvm::createFunctionInliningPass(unsigned OptLevel, unsigned SizeOptLevel) { - return new SimpleInliner( - llvm::computeThresholdFromOptLevels(OptLevel, SizeOptLevel)); + return new SimpleInliner(llvm::getInlineParams(OptLevel, SizeOptLevel)); } bool SimpleInliner::runOnSCC(CallGraphSCC &SCC) {