diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -16,6 +16,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/PriorityQueue.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SetVector.h" @@ -99,6 +100,10 @@ "by inlining from cgscc inline remarks."), cl::Hidden); +static cl::opt InlineEnablePriorityOrder( + "inline-enable-priority-order", cl::Hidden, cl::init(false), + cl::desc("Enable the priority inline order for the inliner")); + LegacyInlinerBase::LegacyInlinerBase(char &ID) : CallGraphSCCPass(ID) {} LegacyInlinerBase::LegacyInlinerBase(char &ID, bool InsertLifetime) @@ -673,6 +678,7 @@ template class InlineOrder { public: using reference = T &; + using const_reference = const T &; virtual ~InlineOrder() {} @@ -680,9 +686,9 @@ virtual void push(const T &Elt) = 0; - virtual void pop() = 0; + virtual T pop() = 0; - virtual reference front() = 0; + virtual const_reference front() = 0; virtual void erase_if(function_ref Pred) = 0; @@ -692,18 +698,19 @@ template > class DefaultInlineOrder : public InlineOrder { using reference = T &; + using const_reference = const T &; public: size_t size() override { return Calls.size() - FirstIndex; } void push(const T &Elt) override { Calls.push_back(Elt); } - void pop() override { + T pop() override { assert(size() > 0); - FirstIndex++; + return Calls[FirstIndex++]; } - reference front() override { + const_reference front() override { assert(size() > 0); return Calls[FirstIndex]; } @@ -718,6 +725,65 @@ size_t FirstIndex = 0; }; +class PriorityInlineOrder : public InlineOrder> { + using T = std::pair; + using reference = T &; + using const_reference = const T &; + + struct Compare { + bool operator()(const T &P1, const T &P2) { return P1.second > P2.second; } + }; + + int evaluate(CallBase *CB) { + Function *Callee = CB->getCalledFunction(); + return (int)Callee->getInstructionCount(); + } + +public: + size_t size() override { return PQ.size(); } + + void push(const T &Elt) override { + CallBase *CB = Elt.first; + const int InlineHistoryID = Elt.second; + const int Goodness = evaluate(CB); + + PQ.push({CB, Goodness}); + InlineHistoryMap[CB] = InlineHistoryID; + } + + T pop() override { + assert(size() > 0); + CallBase *CB = PQ.top().first; + T PopedElt = std::make_pair(CB, InlineHistoryMap[CB]); + InlineHistoryMap.erase(CB); + PQ.pop(); + return PopedElt; + } + + const_reference front() override { + assert(size() > 0); + CallBase *CB = PQ.top().first; + return *InlineHistoryMap.find(CB); + } + + void erase_if(function_ref Pred) override { + PriorityQueue, Compare> TempPQ; + while (!PQ.empty()) { + auto P = PQ.top(); + PQ.pop(); + if (Pred({P.first, 0})) + InlineHistoryMap.erase(P.first); + else + TempPQ.push(P); + } + PQ = std::move(TempPQ); + } + +private: + PriorityQueue, Compare> PQ; + DenseMap InlineHistoryMap; +}; + PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR) { @@ -762,7 +828,12 @@ // this model, but it is uniformly spread across all the functions in the SCC // and eventually they all become too large to inline, rather than // incrementally maknig a single function grow in a super linear fashion. - DefaultInlineOrder> Calls; + std::unique_ptr>> Calls; + if (InlineEnablePriorityOrder) + Calls.reset(new PriorityInlineOrder()); + else + Calls.reset(new DefaultInlineOrder>()); + assert(Calls != nullptr && "Expected an initialized InlineOrder"); // Populate the initial list of calls in this SCC. for (auto &N : InitialC) { @@ -777,7 +848,7 @@ if (auto *CB = dyn_cast(&I)) if (Function *Callee = CB->getCalledFunction()) { if (!Callee->isDeclaration()) - Calls.push({CB, -1}); + Calls->push({CB, -1}); else if (!isa(I)) { using namespace ore; setInlineRemark(*CB, "unavailable definition"); @@ -791,7 +862,7 @@ } } } - if (Calls.empty()) + if (Calls->empty()) return PreservedAnalyses::all(); // Capture updatable variable for the current SCC. @@ -813,15 +884,15 @@ SmallVector DeadFunctions; // Loop forward over all of the calls. - while (!Calls.empty()) { + while (!Calls->empty()) { // We expect the calls to typically be batched with sequences of calls that // have the same caller, so we first set up some shared infrastructure for // this caller. We also do any pruning we can at this layer on the caller // alone. - Function &F = *Calls.front().first->getCaller(); + Function &F = *Calls->front().first->getCaller(); LazyCallGraph::Node &N = *CG.lookup(F); if (CG.lookupSCC(N) != C) { - Calls.pop(); + Calls->pop(); continue; } @@ -837,12 +908,12 @@ // We bail out as soon as the caller has to change so we can update the // call graph and prepare the context of that new caller. bool DidInline = false; - while (!Calls.empty() && Calls.front().first->getCaller() == &F) { - auto &P = Calls.front(); - Calls.pop(); + while (!Calls->empty() && Calls->front().first->getCaller() == &F) { + auto &P = Calls->front(); CallBase *CB = P.first; const int InlineHistoryID = P.second; Function &Callee = *CB->getCalledFunction(); + Calls->pop(); if (InlineHistoryID != -1 && inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) { @@ -909,7 +980,7 @@ } if (NewCallee) if (!NewCallee->isDeclaration()) - Calls.push({ICB, NewHistoryID}); + Calls->push({ICB, NewHistoryID}); } } @@ -926,7 +997,7 @@ // made dead by this operation on other functions). Callee.removeDeadConstantUsers(); if (Callee.use_empty() && !CG.isLibFunction(Callee)) { - Calls.erase_if([&](const std::pair &Call) { + Calls->erase_if([&](const std::pair &Call) { return Call.first->getCaller() == &Callee; }); // Clear the body and queue the function itself for deletion when we