diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -12,6 +12,7 @@ #include "llvm/Analysis/InlineCost.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h" #include #include @@ -65,10 +66,7 @@ /// behavior by implementing the corresponding record*Impl. /// /// Call after inlining succeeded, and did not result in deleting the callee. - void recordInlining() { - markRecorded(); - recordInliningImpl(); - } + void recordInlining(); /// Call after inlining succeeded, and resulted in deleting the callee. void recordInliningWithCalleeDeleted(); @@ -114,6 +112,7 @@ assert(!Recorded && "Recording should happen exactly once"); Recorded = true; } + void recordInlineStatsIfNeeded(); bool Recorded = false; }; @@ -141,7 +140,7 @@ class InlineAdvisor { public: InlineAdvisor(InlineAdvisor &&) = delete; - virtual ~InlineAdvisor() { freeDeletedFunctions(); } + virtual ~InlineAdvisor(); /// Get an InlineAdvice containing a recommendation on whether to /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to @@ -163,12 +162,14 @@ virtual void onPassExit() {} protected: - InlineAdvisor(FunctionAnalysisManager &FAM) : FAM(FAM) {} + InlineAdvisor(Module &M, FunctionAnalysisManager &FAM); virtual std::unique_ptr getAdviceImpl(CallBase &CB) = 0; virtual std::unique_ptr getMandatoryAdvice(CallBase &CB, bool Advice); + Module &M; FunctionAnalysisManager &FAM; + std::unique_ptr ImportedFunctionsStats; /// We may want to defer deleting functions to after the inlining for a whole /// module has finished. This allows us to reliably use function pointers as @@ -202,8 +203,9 @@ /// reusable as-is for inliner pass test scenarios, as well as for regular use. class DefaultInlineAdvisor : public InlineAdvisor { public: - DefaultInlineAdvisor(FunctionAnalysisManager &FAM, InlineParams Params) - : InlineAdvisor(FAM), Params(Params) {} + DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + InlineParams Params) + : InlineAdvisor(M, FAM), Params(Params) {} private: std::unique_ptr getAdviceImpl(CallBase &CB) override; diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h --- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h @@ -50,7 +50,6 @@ virtual std::unique_ptr getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE); - Module &M; std::unique_ptr ModelRunner; private: diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h --- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h @@ -24,8 +24,9 @@ /// previous build to guide current inlining. This is useful for inliner tuning. class ReplayInlineAdvisor : public InlineAdvisor { public: - ReplayInlineAdvisor(FunctionAnalysisManager &FAM, LLVMContext &Context, - StringRef RemarksFile, bool EmitRemarks); + ReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + LLVMContext &Context, StringRef RemarksFile, + bool EmitRemarks); std::unique_ptr getAdviceImpl(CallBase &CB) override; bool areReplayRemarksLoaded() const { return HasReplayRemarks; } diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -97,7 +97,6 @@ class InlinerPass : public PassInfoMixin { public: InlinerPass(bool OnlyMandatory = false) : OnlyMandatory(OnlyMandatory) {} - ~InlinerPass(); InlinerPass(InlinerPass &&Arg) = default; PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, @@ -106,7 +105,6 @@ private: InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM, FunctionAnalysisManager &FAM, Module &M); - std::unique_ptr ImportedFunctionsStats; std::unique_ptr OwnedDefaultAdvisor; const bool OnlyMandatory; }; diff --git a/llvm/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h b/llvm/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h --- a/llvm/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h +++ b/llvm/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h @@ -101,6 +101,12 @@ StringRef ModuleName; }; +enum class InlinerFunctionImportStatsOpts { + No = 0, + Basic = 1, + Verbose = 2, +}; + } // llvm #endif // LLVM_TRANSFORMS_UTILS_IMPORTEDFUNCTIONSINLININGSTATISTICS_H diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -48,6 +48,8 @@ cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden); +extern cl::opt InlinerFunctionImportStats; + void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( const InlineResult &Result) { using namespace ore; @@ -130,8 +132,20 @@ DeletedFunctions.clear(); } +void InlineAdvice::recordInlineStatsIfNeeded() { + if (Advisor->ImportedFunctionsStats) + Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee); +} + +void InlineAdvice::recordInlining() { + markRecorded(); + recordInlineStatsIfNeeded(); + recordInliningImpl(); +} + void InlineAdvice::recordInliningWithCalleeDeleted() { markRecorded(); + recordInlineStatsIfNeeded(); Advisor->markFunctionAsDeleted(Callee); recordInliningWithCalleeDeletedImpl(); } @@ -143,7 +157,7 @@ auto &FAM = MAM.getResult(M).getManager(); switch (Mode) { case InliningAdvisorMode::Default: - Advisor.reset(new DefaultInlineAdvisor(FAM, Params)); + Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params)); break; case InliningAdvisorMode::Development: #ifdef LLVM_HAVE_TF_API @@ -428,6 +442,25 @@ }); } +InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM) + : M(M), FAM(FAM) { + if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { + ImportedFunctionsStats = + std::make_unique(); + ImportedFunctionsStats->setModuleInfo(M); + } +} + +InlineAdvisor::~InlineAdvisor() { + if (ImportedFunctionsStats) { + assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No); + ImportedFunctionsStats->dump(InlinerFunctionImportStats == + InlinerFunctionImportStatsOpts::Verbose); + } + + freeDeletedFunctions(); +} + std::unique_ptr InlineAdvisor::getMandatoryAdvice(CallBase &CB, bool Advice) { return std::make_unique(this, CB, getCallerORE(CB), Advice); diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp --- a/llvm/lib/Analysis/MLInlineAdvisor.cpp +++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp @@ -66,8 +66,8 @@ MLInlineAdvisor::MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::unique_ptr Runner) : InlineAdvisor( - MAM.getResult(M).getManager()), - M(M), ModelRunner(std::move(Runner)), CG(new CallGraph(M)), + M, MAM.getResult(M).getManager()), + ModelRunner(std::move(Runner)), CG(new CallGraph(M)), InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize) { assert(ModelRunner); diff --git a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp --- a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp @@ -22,11 +22,12 @@ #define DEBUG_TYPE "inline-replay" -ReplayInlineAdvisor::ReplayInlineAdvisor(FunctionAnalysisManager &FAM, +ReplayInlineAdvisor::ReplayInlineAdvisor(Module &M, + FunctionAnalysisManager &FAM, LLVMContext &Context, StringRef RemarksFile, bool EmitRemarks) - : InlineAdvisor(FAM), HasReplayRemarks(false), EmitRemarks(EmitRemarks) { + : InlineAdvisor(M, FAM), HasReplayRemarks(false), EmitRemarks(EmitRemarks) { auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(RemarksFile); std::error_code EC = BufferOrErr.getError(); if (EC) { diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -90,24 +90,7 @@ DisableInlinedAllocaMerging("disable-inlined-alloca-merging", cl::init(false), cl::Hidden); -namespace { - -enum class InlinerFunctionImportStatsOpts { - No = 0, - Basic = 1, - Verbose = 2, -}; - -} // end anonymous namespace - -static cl::opt InlinerFunctionImportStats( - "inliner-function-import-stats", - cl::init(InlinerFunctionImportStatsOpts::No), - cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", - "basic statistics"), - clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", - "printing of statistics for each inlined function")), - cl::Hidden, cl::desc("Enable inliner stats for imported functions")); +extern cl::opt InlinerFunctionImportStats; LegacyInlinerBase::LegacyInlinerBase(char &ID) : CallGraphSCCPass(ID) {} @@ -647,17 +630,12 @@ return true; } -InlinerPass::~InlinerPass() { - if (ImportedFunctionsStats) { - assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No); - ImportedFunctionsStats->dump(InlinerFunctionImportStats == - InlinerFunctionImportStatsOpts::Verbose); - } -} - InlineAdvisor & InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM, FunctionAnalysisManager &FAM, Module &M) { + if (OwnedDefaultAdvisor) + return *OwnedDefaultAdvisor; + auto *IAA = MAM.getCachedResult(M); if (!IAA) { // It should still be possible to run the inliner as a stand-alone SCC pass, @@ -669,7 +647,7 @@ // The one we would get from the MAM can be invalidated as a result of the // inliner's activity. OwnedDefaultAdvisor = - std::make_unique(FAM, getInlineParams()); + std::make_unique(M, FAM, getInlineParams()); return *OwnedDefaultAdvisor; } assert(IAA->getAdvisor() && @@ -698,13 +676,6 @@ auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(); }); - if (!ImportedFunctionsStats && - InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { - ImportedFunctionsStats = - std::make_unique(); - ImportedFunctionsStats->setModuleInfo(M); - } - // We use a single common worklist for calls across the entire SCC. We // process these in-order and append new calls introduced during inlining to // the end. @@ -879,9 +850,6 @@ } } - if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) - ImportedFunctionsStats->recordInline(F, Callee); - // Merge the attributes based on the inlining. AttributeFuncs::mergeAttributesForInlining(F, Callee); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -1962,7 +1962,7 @@ if (FAM && !ProfileInlineReplayFile.empty()) { ExternalInlineAdvisor = std::make_unique( - *FAM, Ctx, ProfileInlineReplayFile, /*EmitRemarks=*/false); + M, *FAM, Ctx, ProfileInlineReplayFile, /*EmitRemarks=*/false); if (!ExternalInlineAdvisor->areReplayRemarksLoaded()) ExternalInlineAdvisor.reset(); } diff --git a/llvm/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp b/llvm/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp --- a/llvm/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp +++ b/llvm/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp @@ -13,6 +13,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include @@ -20,6 +21,15 @@ #include using namespace llvm; +cl::opt InlinerFunctionImportStats( + "inliner-function-import-stats", + cl::init(InlinerFunctionImportStatsOpts::No), + cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", + "basic statistics"), + clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", + "printing of statistics for each inlined function")), + cl::Hidden, cl::desc("Enable inliner stats for imported functions")); + ImportedFunctionsInliningStatistics::InlineGraphNode & ImportedFunctionsInliningStatistics::createInlineGraphNode(const Function &F) { diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll --- a/llvm/test/Transforms/Inline/inline_stats.ll +++ b/llvm/test/Transforms/Inline/inline_stats.ll @@ -9,6 +9,9 @@ ; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK ; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK +; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK +; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-VERBOSE,CHECK + ; CHECK: ------- Dumping inliner stats for [] ------- ; CHECK-BASIC-NOT: -- List of inlined functions: ; CHECK-BASIC-NOT: -- Inlined not imported function