Index: include/llvm/Analysis/InlineCost.h =================================================================== --- include/llvm/Analysis/InlineCost.h +++ include/llvm/Analysis/InlineCost.h @@ -116,14 +116,15 @@ /// \brief Get an InlineCost object representing the cost of inlining this /// callsite. /// - /// Note that threshold is passed into this function. Only costs below the - /// threshold are computed with any accuracy. The threshold can be used to + /// Note that a threshold is passed into this function. This threshold is + /// modified based on callsite's properties and only costs below this new + /// threshold are computed with any accuracy. The new threshold can be used to /// bound the computation necessary to determine whether the cost is /// sufficiently low to warrant inlining. /// /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. - InlineCost getInlineCost(CallSite CS, int Threshold); + InlineCost getInlineCost(CallSite CS, int CallSiteIndependentThreshold); /// \brief Get an InlineCost with the callee explicitly specified. /// This allows you to calculate the cost of inlining a function via a @@ -132,12 +133,19 @@ // // Note: This is used by out-of-tree passes, please do not remove without // adding a replacement API. - InlineCost getInlineCost(CallSite CS, Function *Callee, int Threshold); + InlineCost getInlineCost(CallSite CS, Function *Callee, + int CallSiteIndependentThreshold); /// \brief Minimal filter to detect invalid constructs for inlining. bool isInlineViable(Function &Callee); + }; +/// \brief Compute threshold for a given optimization level. +int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel); + +/// \brief Return the default value of -inline-threshold. +int getDefaultInlineThreshold(); } #endif Index: include/llvm/Transforms/IPO/InlinerPass.h =================================================================== --- include/llvm/Transforms/IPO/InlinerPass.h +++ include/llvm/Transforms/IPO/InlinerPass.h @@ -31,7 +31,7 @@ /// struct Inliner : public CallGraphSCCPass { explicit Inliner(char &ID); - explicit Inliner(char &ID, int Threshold, bool InsertLifetime); + explicit Inliner(char &ID, bool InsertLifetime); /// getAnalysisUsage - For this class, we declare that we require and preserve /// the call graph. If the derived class implements this method, it should @@ -47,18 +47,6 @@ // processing to avoid breaking the SCC traversal. bool doFinalization(CallGraph &CG) override; - /// This method returns the value specified by the -inline-threshold value, - /// specified on the command line. This is typically not directly needed. - /// - unsigned getInlineThreshold() const { return InlineThreshold; } - - /// Calculate the inline threshold for given Caller. This threshold is lower - /// if the caller is marked with OptimizeForSize and -inline-threshold is not - /// given on the comand line. It is higher if the callee is marked with the - /// inlinehint attribute. - /// - unsigned getInlineThreshold(CallSite CS) const; - /// getInlineCost - This method must be implemented by the subclass to /// determine the cost of inlining the specified call site. If the cost /// returned is greater than the current inline threshold, the call site is @@ -75,9 +63,6 @@ bool removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly = false); private: - // InlineThreshold - Cache the value here for easy access. - unsigned InlineThreshold; - // InsertLifetime - Insert @llvm.lifetime intrinsics. bool InsertLifetime; Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -1345,8 +1345,10 @@ return false; } -InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, int Threshold) { - return getInlineCost(CS, CS.getCalledFunction(), Threshold); +InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, + int CallSiteIndependentThreshold) { + return getInlineCost(CS, CS.getCalledFunction(), + CallSiteIndependentThreshold); } /// \brief Test that two functions either have or have not the given attribute @@ -1367,8 +1369,88 @@ attributeMatches(Caller, Callee, Attribute::SanitizeThread); } +// Inline threshold computation + +// Threshold to use when optsize is specified (and there is no +// -inline-threshold). +const int OptSizeThreshold = 75; + +// Threshold to use when -Oz is specified (and there is no -inline-threshold). +const int OptMinSizeThreshold = 25; + +// Threshold to use when -O[34] is specified (and there is no +// -inline-threshold). +const int OptAggressiveThreshold = 275; + +static cl::opt InlineLimit( + "inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, + cl::desc("Control the amount of inlining to perform (default = 225)")); + +static cl::opt HintThreshold( + "inlinehint-threshold", cl::Hidden, cl::init(325), + cl::desc("Threshold for inlining functions with inline hint")); + +// We introduce this threshold to help performance of instrumentation based +// PGO before we actually hook up inliner with analysis passes such as BPI and +// BFI. +static cl::opt ColdThreshold( + "inlinecold-threshold", cl::Hidden, cl::init(225), + cl::desc("Threshold for inlining functions with cold attribute")); + +namespace llvm { +int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel) { + if (OptLevel > 2) + return OptAggressiveThreshold; + if (SizeOptLevel == 1) // -Os + return OptSizeThreshold; + if (SizeOptLevel == 2) // -Oz + return OptMinSizeThreshold; + return InlineLimit; +} + +int getDefaultInlineThreshold() { return InlineLimit; } +} + +static int getInlineThreshold(CallSite CS, int CallSiteIndependentThreshold) { + + int Threshold = CallSiteIndependentThreshold; + // If -inline-threshold is not given, listen to the optsize attribute when it + // would decrease the threshold. + Function *Caller = CS.getCaller(); + bool OptSize = Caller && !Caller->isDeclaration() && + // FIXME: Use Function::optForSize(). + Caller->hasFnAttribute(Attribute::OptimizeForSize); + if (!(InlineLimit.getNumOccurrences() > 0) && OptSize && + OptSizeThreshold < Threshold) + Threshold = OptSizeThreshold; + + Function *Callee = CS.getCalledFunction(); + if (!Callee || Callee->isDeclaration()) + return Threshold; + + // Listen to the inlinehint attribute when it would increase the threshold + // and the caller does not need to minimize its size. + bool InlineHint = Callee->hasFnAttribute(Attribute::InlineHint); + if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) + Threshold = HintThreshold; + + // Listen to the cold attribute when it would decrease the threshold. + bool ColdCallee = Callee->hasFnAttribute(Attribute::Cold); + // Command line argument for InlineLimit will override the default + // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, + // do not use the default cold threshold even if it is smaller. + if ((InlineLimit.getNumOccurrences() == 0 || + ColdThreshold.getNumOccurrences() > 0) && + ColdCallee && ColdThreshold < Threshold) + Threshold = ColdThreshold; + + return Threshold; +} + InlineCost InlineCostAnalysis::getInlineCost(CallSite CS, Function *Callee, - int Threshold) { + int CallSiteIndependentThreshold) { + int Threshold = getInlineThreshold(CS, CallSiteIndependentThreshold); + // Cannot inline indirect calls. if (!Callee) return llvm::InlineCost::getNever(); Index: lib/Transforms/IPO/InlineAlways.cpp =================================================================== --- lib/Transforms/IPO/InlineAlways.cpp +++ lib/Transforms/IPO/InlineAlways.cpp @@ -38,14 +38,12 @@ InlineCostAnalysis *ICA; public: - // Use extremely low threshold. - AlwaysInliner() : Inliner(ID, -2000000000, /*InsertLifetime*/ true), - ICA(nullptr) { + AlwaysInliner() : Inliner(ID, /*InsertLifetime*/ true), ICA(nullptr) { initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry()); } AlwaysInliner(bool InsertLifetime) - : Inliner(ID, -2000000000, InsertLifetime), ICA(nullptr) { + : Inliner(ID, InsertLifetime), ICA(nullptr) { initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry()); } Index: lib/Transforms/IPO/InlineSimple.cpp =================================================================== --- lib/Transforms/IPO/InlineSimple.cpp +++ lib/Transforms/IPO/InlineSimple.cpp @@ -38,38 +38,29 @@ /// analyses to determine when to inline. class SimpleInliner : public Inliner { InlineCostAnalysis *ICA; + int Threshold; public: - SimpleInliner() : Inliner(ID), ICA(nullptr) { + SimpleInliner() + : Inliner(ID), ICA(nullptr), Threshold(getDefaultInlineThreshold()) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } SimpleInliner(int Threshold) - : Inliner(ID, Threshold, /*InsertLifetime*/ true), ICA(nullptr) { + : Inliner(ID), ICA(nullptr), Threshold(Threshold) { initializeSimpleInlinerPass(*PassRegistry::getPassRegistry()); } static char ID; // Pass identification, replacement for typeid InlineCost getInlineCost(CallSite CS) override { - return ICA->getInlineCost(CS, getInlineThreshold(CS)); + return ICA->getInlineCost(CS, Threshold); } bool runOnSCC(CallGraphSCC &SCC) override; void getAnalysisUsage(AnalysisUsage &AU) const override; }; -static int computeThresholdFromOptLevels(unsigned OptLevel, - unsigned SizeOptLevel) { - if (OptLevel > 2) - return 275; - if (SizeOptLevel == 1) // -Os - return 75; - if (SizeOptLevel == 2) // -Oz - return 25; - return 225; -} - } // end anonymous namespace char SimpleInliner::ID = 0; Index: lib/Transforms/IPO/Inliner.cpp =================================================================== --- lib/Transforms/IPO/Inliner.cpp +++ lib/Transforms/IPO/Inliner.cpp @@ -47,31 +47,10 @@ // if those would be more profitable and blocked inline steps. STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed"); -static cl::opt -InlineLimit("inline-threshold", cl::Hidden, cl::init(225), cl::ZeroOrMore, - cl::desc("Control the amount of inlining to perform (default = 225)")); +Inliner::Inliner(char &ID) : CallGraphSCCPass(ID), InsertLifetime(true) {} -static cl::opt -HintThreshold("inlinehint-threshold", cl::Hidden, cl::init(325), - cl::desc("Threshold for inlining functions with inline hint")); - -// We instroduce this threshold to help performance of instrumentation based -// PGO before we actually hook up inliner with analysis passes such as BPI and -// BFI. -static cl::opt -ColdThreshold("inlinecold-threshold", cl::Hidden, cl::init(225), - cl::desc("Threshold for inlining functions with cold attribute")); - -// Threshold to use when optsize is specified (and there is no -inline-limit). -const int OptSizeThreshold = 75; - -Inliner::Inliner(char &ID) - : CallGraphSCCPass(ID), InlineThreshold(InlineLimit), InsertLifetime(true) {} - -Inliner::Inliner(char &ID, int Threshold, bool InsertLifetime) - : CallGraphSCCPass(ID), InlineThreshold(InlineLimit.getNumOccurrences() > 0 ? - InlineLimit : Threshold), - InsertLifetime(InsertLifetime) {} +Inliner::Inliner(char &ID, bool InsertLifetime) + : CallGraphSCCPass(ID), InsertLifetime(InsertLifetime) {} /// For this class, we declare that we require and preserve the call graph. /// If the derived class implements this method, it should @@ -274,43 +253,6 @@ return true; } -unsigned Inliner::getInlineThreshold(CallSite CS) const { - int Threshold = InlineThreshold; // -inline-threshold or else selected by - // overall opt level - - // If -inline-threshold is not given, listen to the optsize attribute when it - // would decrease the threshold. - Function *Caller = CS.getCaller(); - bool OptSize = Caller && !Caller->isDeclaration() && - // FIXME: Use Function::optForSize(). - Caller->hasFnAttribute(Attribute::OptimizeForSize); - if (!(InlineLimit.getNumOccurrences() > 0) && OptSize && - OptSizeThreshold < Threshold) - Threshold = OptSizeThreshold; - - // Listen to the inlinehint attribute when it would increase the threshold - // and the caller does not need to minimize its size. - Function *Callee = CS.getCalledFunction(); - bool InlineHint = Callee && !Callee->isDeclaration() && - Callee->hasFnAttribute(Attribute::InlineHint); - if (InlineHint && HintThreshold > Threshold && - !Caller->hasFnAttribute(Attribute::MinSize)) - Threshold = HintThreshold; - - // Listen to the cold attribute when it would decrease the threshold. - bool ColdCallee = Callee && !Callee->isDeclaration() && - Callee->hasFnAttribute(Attribute::Cold); - // Command line argument for InlineLimit will override the default - // ColdThreshold. If we have -inline-threshold but no -inlinecold-threshold, - // do not use the default cold threshold even if it is smaller. - if ((InlineLimit.getNumOccurrences() == 0 || - ColdThreshold.getNumOccurrences() > 0) && ColdCallee && - ColdThreshold < Threshold) - Threshold = ColdThreshold; - - return Threshold; -} - static void emitAnalysis(CallSite CS, const Twine &Msg) { Function *Caller = CS.getCaller(); LLVMContext &Ctx = Caller->getContext();