diff --git a/llvm/include/llvm/Analysis/InlineOrder.h b/llvm/include/llvm/Analysis/InlineOrder.h
--- a/llvm/include/llvm/Analysis/InlineOrder.h
+++ b/llvm/include/llvm/Analysis/InlineOrder.h
@@ -12,6 +12,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/InlineCost.h"
#include "llvm/IR/InstrTypes.h"
#include <algorithm>
#include <utility>
@@ -20,6 +21,8 @@
class CallBase;
class Function;
+enum class InlinePriorityMode : int { NoPriority, Size, Cost, OptRatio };
+
template <typename T> class InlineOrder {
public:
using reference = T &;
@@ -40,6 +43,10 @@
bool empty() { return !size(); }
};
+std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
+getInlineOrder(InlinePriorityMode UseInlinePriority,
+ FunctionAnalysisManager &FAM, const InlineParams &Params);
+
template <typename T, typename Container = SmallVector<T, 16>>
class DefaultInlineOrder : public InlineOrder<T> {
using reference = T &;
@@ -82,15 +89,16 @@
using PriorityT = unsigned;
DenseMap<const CallBase *, PriorityT> Priorities;
- static PriorityT evaluate(const CallBase *CB) {
+ PriorityT evaluate(const CallBase *CB) {
Function *Callee = CB->getCalledFunction();
return Callee->getInstructionCount();
}
- static bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) {
+ bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) const {
return P1 < P2;
}
+public:
bool hasLowerPriority(const CallBase *L, const CallBase *R) const override {
const auto I1 = Priorities.find(L);
const auto I2 = Priorities.find(R);
@@ -98,7 +106,49 @@
return isMoreDesirable(I2->second, I1->second);
}
+ // Update the priority associated with CB.
+ void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); };
+
+ bool updateAndCheckDecreased(const CallBase *CB) override {
+ auto It = Priorities.find(CB);
+ const auto OldPriority = It->second;
+ It->second = evaluate(CB);
+ const auto NewPriority = It->second;
+ return isMoreDesirable(OldPriority, NewPriority);
+ }
+};
+
+class CostPriority : public InlinePriority {
+ using PriorityT = int;
+ DenseMap<const CallBase *, PriorityT> Priorities;
+ std::function<InlineCost(const CallBase *)> getInlineCost;
+
+ PriorityT evaluate(const CallBase *CB) {
+ auto IC = getInlineCost(CB);
+ int cost = 0;
+ if (IC.isVariable())
+ cost = IC.getCost();
+ else
+ cost = IC.isNever() ? INT_MAX : INT_MIN;
+ return cost;
+ }
+
+ bool isMoreDesirable(const PriorityT &P1, const PriorityT &P2) const {
+ return P1 < P2;
+ }
+
public:
+ CostPriority() = delete;
+ CostPriority(std::function<InlineCost(const CallBase *)> getInlineCost)
+ : getInlineCost(getInlineCost){};
+
+ bool hasLowerPriority(const CallBase *L, const CallBase *R) const override {
+ const auto I1 = Priorities.find(L);
+ const auto I2 = Priorities.find(R);
+ assert(I1 != Priorities.end() && I2 != Priorities.end());
+ return isMoreDesirable(I2->second, I1->second);
+ }
+
// Update the priority associated with CB.
void update(const CallBase *CB) override { Priorities[CB] = evaluate(CB); };
@@ -184,5 +234,6 @@
DenseMap<CallBase *, int> InlineHistoryMap;
std::unique_ptr<InlinePriority> PriorityPtr;
};
+
} // namespace llvm
#endif // LLVM_ANALYSIS_INLINEORDER_H
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -75,6 +75,7 @@
IndirectCallPromotionAnalysis.cpp
InlineCost.cpp
InlineAdvisor.cpp
+ InlineOrder.cpp
InlineSizeEstimatorAnalysis.cpp
InstCount.cpp
InstructionPrecedenceTracking.cpp
diff --git a/llvm/lib/Analysis/InlineOrder.cpp b/llvm/lib/Analysis/InlineOrder.cpp
new file mode 100644
--- /dev/null
+++ b/llvm/lib/Analysis/InlineOrder.cpp
@@ -0,0 +1,69 @@
+#include "llvm/Analysis/InlineOrder.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/InlineAdvisor.h"
+#include "llvm/Analysis/InlineCost.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "inline-order"
+
+static llvm::InlineCost getInlineCostWrapper(CallBase &CB,
+ FunctionAnalysisManager &FAM,
+ const InlineParams &Params) {
+ Function &Caller = *CB.getCaller();
+ ProfileSummaryInfo *PSI =
+ FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
+ .getCachedResult<ProfileSummaryAnalysis>(
+ *CB.getParent()->getParent()->getParent());
+
+ auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
+ auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
+ return FAM.getResult<AssumptionAnalysis>(F);
+ };
+ auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
+ return FAM.getResult<BlockFrequencyAnalysis>(F);
+ };
+ auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
+ return FAM.getResult<TargetLibraryAnalysis>(F);
+ };
+
+ Function &Callee = *CB.getCalledFunction();
+ auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
+ bool RemarksEnabled =
+ Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
+ DEBUG_TYPE);
+ return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
+ GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
+}
+
+std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
+llvm::getInlineOrder(InlinePriorityMode UseInlinePriority,
+ FunctionAnalysisManager &FAM, const InlineParams &Params) {
+ switch (UseInlinePriority) {
+ case InlinePriorityMode::NoPriority:
+ return std::make_unique<DefaultInlineOrder<std::pair<CallBase *, int>>>();
+
+ case InlinePriorityMode::Size:
+ LLVM_DEBUG(dbgs() << " Current used priority: Size priority ---- \n");
+ return std::make_unique<PriorityInlineOrder>(
+ std::make_unique<SizePriority>());
+
+ case InlinePriorityMode::Cost:
+ LLVM_DEBUG(dbgs() << " Current used priority: Cost priority ---- \n");
+ return std::make_unique<PriorityInlineOrder>(
+ std::make_unique<CostPriority>([&](const CallBase *CB) -> InlineCost {
+ return getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);
+ }));
+
+ default:
+ assert("Unsupported Inline Priority Mode");
+ break;
+ }
+ return nullptr;
+}
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -49,9 +49,15 @@
STATISTIC(NumInlined, "Number of functions inlined");
STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
-static cl::opt<bool> InlineEnablePriorityOrder(
- "module-inline-enable-priority-order", cl::Hidden, cl::init(true),
- cl::desc("Enable the priority inline order for the module inliner"));
+static cl::opt<InlinePriorityMode> UseInlinePriority(
+ "inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden,
+ cl::desc("Choose the priority mode to use in module inline"),
+ cl::values(clEnumValN(InlinePriorityMode::NoPriority, "no priority",
+ "Use no priority, visit callsites in bottom-up."),
+ clEnumValN(InlinePriorityMode::Size, "size",
+ "Use callee size priority."),
+ clEnumValN(InlinePriorityMode::Cost, "cost",
+ "Use inline cost priority.")));
/// Return true if the specified inline history ID
/// indicates an inline history that includes the specified function.
@@ -146,11 +152,7 @@
// TODO: Here is a huge amount duplicate code between the module inliner and
// the SCC inliner, which need some refactoring.
std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>> Calls;
- if (InlineEnablePriorityOrder)
- Calls = std::make_unique<PriorityInlineOrder>(
- std::make_unique<SizePriority>());
- else
- Calls = std::make_unique<DefaultInlineOrder<std::pair<CallBase *, int>>>();
+ Calls = getInlineOrder(UseInlinePriority, FAM, Params);
assert(Calls != nullptr && "Expected an initialized InlineOrder");
// Populate the initial list of calls in this module.