diff --git a/llvm/include/llvm/Analysis/InlineOrder.h b/llvm/include/llvm/Analysis/InlineOrder.h --- a/llvm/include/llvm/Analysis/InlineOrder.h +++ b/llvm/include/llvm/Analysis/InlineOrder.h @@ -77,163 +77,5 @@ size_t FirstIndex = 0; }; -class InlinePriority { -public: - virtual ~InlinePriority() = default; - virtual bool hasLowerPriority(const CallBase *L, const CallBase *R) const = 0; - virtual void update(const CallBase *CB) = 0; - virtual bool updateAndCheckDecreased(const CallBase *CB) = 0; -}; - -class SizePriority : public InlinePriority { - using PriorityT = unsigned; - DenseMap Priorities; - - PriorityT evaluate(const CallBase *CB) { - Function *Callee = CB->getCalledFunction(); - return Callee->getInstructionCount(); - } - - bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) const { - return P1 < P2; - } - -public: - bool hasLowerPriority(const CallBase *L, const CallBase *R) const override { - const auto I1 = Priorities.find(L); - const auto I2 = Priorities.find(R); - assert(I1 != Priorities.end() && I2 != Priorities.end()); - return isMoreDesirable(I2->second, I1->second); - } - - // Update the priority associated with CB. - void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); }; - - bool updateAndCheckDecreased(const CallBase *CB) override { - auto It = Priorities.find(CB); - const auto OldPriority = It->second; - It->second = evaluate(CB); - const auto NewPriority = It->second; - return isMoreDesirable(OldPriority, NewPriority); - } -}; - -class CostPriority : public InlinePriority { - using PriorityT = int; - DenseMap Priorities; - std::function getInlineCost; - - PriorityT evaluate(const CallBase *CB) { - auto IC = getInlineCost(CB); - int cost = 0; - if (IC.isVariable()) - cost = IC.getCost(); - else - cost = IC.isNever() ? INT_MAX : INT_MIN; - return cost; - } - - bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) const { - return P1 < P2; - } - -public: - CostPriority() = delete; - CostPriority(std::function getInlineCost) - : getInlineCost(getInlineCost){}; - - bool hasLowerPriority(const CallBase *L, const CallBase *R) const override { - const auto I1 = Priorities.find(L); - const auto I2 = Priorities.find(R); - assert(I1 != Priorities.end() && I2 != Priorities.end()); - return isMoreDesirable(I2->second, I1->second); - } - - // Update the priority associated with CB. - void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); }; - - bool updateAndCheckDecreased(const CallBase *CB) override { - auto It = Priorities.find(CB); - const auto OldPriority = It->second; - It->second = evaluate(CB); - const auto NewPriority = It->second; - return isMoreDesirable(OldPriority, NewPriority); - } -}; - -class PriorityInlineOrder : public InlineOrder> { - using T = std::pair; - using reference = T &; - using const_reference = const T &; - - // A call site could become less desirable for inlining because of the size - // growth from prior inlining into the callee. This method is used to lazily - // update the desirability of a call site if it's decreasing. It is only - // called on pop() or front(), not every time the desirability changes. When - // the desirability of the front call site decreases, an updated one would be - // pushed right back into the heap. For simplicity, those cases where - // the desirability of a call site increases are ignored here. - void adjust() { - while (PriorityPtr->updateAndCheckDecreased(Heap.front())) { - std::pop_heap(Heap.begin(), Heap.end(), isLess); - std::push_heap(Heap.begin(), Heap.end(), isLess); - } - } - -public: - PriorityInlineOrder(std::unique_ptr PriorityPtr) - : PriorityPtr(std::move(PriorityPtr)) { - isLess = [this](const CallBase *L, const CallBase *R) { - return this->PriorityPtr->hasLowerPriority(L, R); - }; - } - - size_t size() override { return Heap.size(); } - - void push(const T &Elt) override { - CallBase *CB = Elt.first; - const int InlineHistoryID = Elt.second; - - Heap.push_back(CB); - PriorityPtr->update(CB); - std::push_heap(Heap.begin(), Heap.end(), isLess); - InlineHistoryMap[CB] = InlineHistoryID; - } - - T pop() override { - assert(size() > 0); - adjust(); - - CallBase *CB = Heap.front(); - T Result = std::make_pair(CB, InlineHistoryMap[CB]); - InlineHistoryMap.erase(CB); - std::pop_heap(Heap.begin(), Heap.end(), isLess); - Heap.pop_back(); - return Result; - } - - const_reference front() override { - assert(size() > 0); - adjust(); - - CallBase *CB = Heap.front(); - return *InlineHistoryMap.find(CB); - } - - void erase_if(function_ref Pred) override { - auto PredWrapper = [=](CallBase *CB) -> bool { - return Pred(std::make_pair(CB, 0)); - }; - llvm::erase_if(Heap, PredWrapper); - std::make_heap(Heap.begin(), Heap.end(), isLess); - } - -private: - SmallVector Heap; - std::function isLess; - DenseMap InlineHistoryMap; - std::unique_ptr PriorityPtr; -}; - } // namespace llvm #endif // LLVM_ANALYSIS_INLINEORDER_H diff --git a/llvm/lib/Analysis/InlineOrder.cpp b/llvm/lib/Analysis/InlineOrder.cpp --- a/llvm/lib/Analysis/InlineOrder.cpp +++ b/llvm/lib/Analysis/InlineOrder.cpp @@ -21,6 +21,168 @@ #define DEBUG_TYPE "inline-order" +namespace { + +class InlinePriority { +public: + virtual ~InlinePriority() = default; + virtual bool hasLowerPriority(const CallBase *L, const CallBase *R) const = 0; + virtual void update(const CallBase *CB) = 0; + virtual bool updateAndCheckDecreased(const CallBase *CB) = 0; +}; + +class SizePriority : public InlinePriority { + using PriorityT = unsigned; + DenseMap Priorities; + + PriorityT evaluate(const CallBase *CB) { + Function *Callee = CB->getCalledFunction(); + return Callee->getInstructionCount(); + } + + bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) const { + return P1 < P2; + } + +public: + bool hasLowerPriority(const CallBase *L, const CallBase *R) const override { + const auto I1 = Priorities.find(L); + const auto I2 = Priorities.find(R); + assert(I1 != Priorities.end() && I2 != Priorities.end()); + return isMoreDesirable(I2->second, I1->second); + } + + // Update the priority associated with CB. + void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); }; + + bool updateAndCheckDecreased(const CallBase *CB) override { + auto It = Priorities.find(CB); + const auto OldPriority = It->second; + It->second = evaluate(CB); + const auto NewPriority = It->second; + return isMoreDesirable(OldPriority, NewPriority); + } +}; + +class CostPriority : public InlinePriority { + using PriorityT = int; + DenseMap Priorities; + std::function getInlineCost; + + PriorityT evaluate(const CallBase *CB) { + auto IC = getInlineCost(CB); + int cost = 0; + if (IC.isVariable()) + cost = IC.getCost(); + else + cost = IC.isNever() ? INT_MAX : INT_MIN; + return cost; + } + + bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) const { + return P1 < P2; + } + +public: + CostPriority() = delete; + CostPriority(std::function getInlineCost) + : getInlineCost(getInlineCost){}; + + bool hasLowerPriority(const CallBase *L, const CallBase *R) const override { + const auto I1 = Priorities.find(L); + const auto I2 = Priorities.find(R); + assert(I1 != Priorities.end() && I2 != Priorities.end()); + return isMoreDesirable(I2->second, I1->second); + } + + // Update the priority associated with CB. + void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); }; + + bool updateAndCheckDecreased(const CallBase *CB) override { + auto It = Priorities.find(CB); + const auto OldPriority = It->second; + It->second = evaluate(CB); + const auto NewPriority = It->second; + return isMoreDesirable(OldPriority, NewPriority); + } +}; + +class PriorityInlineOrder : public InlineOrder> { + using T = std::pair; + using reference = T &; + using const_reference = const T &; + + // A call site could become less desirable for inlining because of the size + // growth from prior inlining into the callee. This method is used to lazily + // update the desirability of a call site if it's decreasing. It is only + // called on pop() or front(), not every time the desirability changes. When + // the desirability of the front call site decreases, an updated one would be + // pushed right back into the heap. For simplicity, those cases where + // the desirability of a call site increases are ignored here. + void adjust() { + while (PriorityPtr->updateAndCheckDecreased(Heap.front())) { + std::pop_heap(Heap.begin(), Heap.end(), isLess); + std::push_heap(Heap.begin(), Heap.end(), isLess); + } + } + +public: + PriorityInlineOrder(std::unique_ptr PriorityPtr) + : PriorityPtr(std::move(PriorityPtr)) { + isLess = [this](const CallBase *L, const CallBase *R) { + return this->PriorityPtr->hasLowerPriority(L, R); + }; + } + + size_t size() override { return Heap.size(); } + + void push(const T &Elt) override { + CallBase *CB = Elt.first; + const int InlineHistoryID = Elt.second; + + Heap.push_back(CB); + PriorityPtr->update(CB); + std::push_heap(Heap.begin(), Heap.end(), isLess); + InlineHistoryMap[CB] = InlineHistoryID; + } + + T pop() override { + assert(size() > 0); + adjust(); + + CallBase *CB = Heap.front(); + T Result = std::make_pair(CB, InlineHistoryMap[CB]); + InlineHistoryMap.erase(CB); + std::pop_heap(Heap.begin(), Heap.end(), isLess); + Heap.pop_back(); + return Result; + } + + const_reference front() override { + assert(size() > 0); + adjust(); + + CallBase *CB = Heap.front(); + return *InlineHistoryMap.find(CB); + } + + void erase_if(function_ref Pred) override { + auto PredWrapper = [=](CallBase *CB) -> bool { + return Pred(std::make_pair(CB, 0)); + }; + llvm::erase_if(Heap, PredWrapper); + std::make_heap(Heap.begin(), Heap.end(), isLess); + } + +private: + SmallVector Heap; + std::function isLess; + DenseMap InlineHistoryMap; + std::unique_ptr PriorityPtr; +}; + +} // namespace + static llvm::InlineCost getInlineCostWrapper(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {