diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -143,7 +143,11 @@ /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates /// only mandatory (always-inline) call sites should be recommended - this /// allows the InlineAdvisor track such inlininings. - /// Returns an InlineAdvice with the inlining recommendation. + /// Returns: + /// - An InlineAdvice with the inlining recommendation. + /// - Null when no recommendation is made (https://reviews.llvm.org/D110658). + /// TODO: Consider removing the Null return scenario by incorporating the + /// SampleProfile inliner into an InlineAdvisor std::unique_ptr getAdvice(CallBase &CB, bool MandatoryOnly = false); @@ -225,7 +229,7 @@ return false; } bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, - StringRef ReplayFile); + StringRef ReplayFile, StringRef ReplayScope); InlineAdvisor *getAdvisor() const { return Advisor.get(); } void clear() { Advisor.reset(); } diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h --- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h @@ -20,6 +20,8 @@ class Module; class OptimizationRemarkEmitter; +enum class ReplayInlineScope { Function, Module }; + /// Replay inline advisor that uses optimization remarks from inlining of /// previous build to guide current inlining. This is useful for inliner tuning. class ReplayInlineAdvisor : public InlineAdvisor { @@ -27,15 +29,20 @@ ReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr OriginalAdvisor, - StringRef RemarksFile, bool EmitRemarks); + StringRef RemarksFile, StringRef ScopeString, + bool EmitRemarks); std::unique_ptr getAdviceImpl(CallBase &CB) override; bool areReplayRemarksLoaded() const { return HasReplayRemarks; } private: - StringSet<> InlineSitesFromRemarks; std::unique_ptr OriginalAdvisor; bool HasReplayRemarks = false; bool EmitRemarks = false; + + ReplayInlineScope Scope; + + StringMap InlineSitesFromRemarks; + StringSet<> InlineCallersFromRemarks; }; } // namespace llvm #endif // LLVM_ANALYSIS_REPLAYINLINEADVISOR_H diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -153,7 +153,8 @@ bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params, InliningAdvisorMode Mode, - StringRef ReplayFile) { + StringRef ReplayFile, + StringRef ReplayScope) { auto &FAM = MAM.getResult(M).getManager(); switch (Mode) { case InliningAdvisorMode::Default: @@ -163,7 +164,7 @@ // replay will need augmentations to interleave with them correctly. if (!ReplayFile.empty()) { Advisor = std::make_unique( - M, FAM, M.getContext(), std::move(Advisor), ReplayFile, + M, FAM, M.getContext(), std::move(Advisor), ReplayFile, ReplayScope, /* EmitRemarks =*/true); } break; diff --git a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp --- a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp @@ -25,9 +25,20 @@ ReplayInlineAdvisor::ReplayInlineAdvisor( Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr OriginalAdvisor, StringRef RemarksFile, - bool EmitRemarks) + StringRef ScopeString, bool EmitRemarks) : InlineAdvisor(M, FAM), OriginalAdvisor(std::move(OriginalAdvisor)), HasReplayRemarks(false), EmitRemarks(EmitRemarks) { + + if (ScopeString == "Module" || ScopeString == "") + Scope = ReplayInlineScope::Module; + else if (ScopeString == "Function") + Scope = ReplayInlineScope::Function; + else { + Context.emitError( + "Invalid argument for replay scope, must be 'Module' or 'Function'"); + return; + } + auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(RemarksFile); std::error_code EC = BufferOrErr.getError(); if (EC) { @@ -36,23 +47,30 @@ } // Example for inline remarks to parse: - // main:3:1.1: '_Z3subii' inlined into 'main' at callsite sum:1 @ main:3:1.1 + // main:3:1.1: '_Z3subii' inlined into 'main' at callsite sum:1 @ + // main:3:1.1; // We use the callsite string after `at callsite` to replay inlining. line_iterator LineIt(*BufferOrErr.get(), /*SkipBlanks=*/true); for (; !LineIt.is_at_eof(); ++LineIt) { StringRef Line = *LineIt; auto Pair = Line.split(" at callsite "); - StringRef Callee = Pair.first.split(" inlined into") - .first.rsplit(": '") - .second.drop_back(); + auto CalleeCaller = Pair.first.split("' inlined into '"); + + StringRef Callee = CalleeCaller.first.rsplit(": '").second; + StringRef Caller = CalleeCaller.second.rsplit("'").first; + auto CallSite = Pair.second.split(";").first; - if (Callee.empty() || CallSite.empty()) - continue; + if (Callee.empty() || Caller.empty() || CallSite.empty()) { + Context.emitError("Invalid remark format: " + Line); + return; + } std::string Combined = (Callee + CallSite).str(); - InlineSitesFromRemarks.insert(Combined); + InlineSitesFromRemarks[Combined] = false; + if (Scope == ReplayInlineScope::Function) + InlineCallersFromRemarks.insert(Caller); } HasReplayRemarks = true; @@ -64,18 +82,23 @@ Function &Caller = *CB.getCaller(); auto &ORE = FAM.getResult(Caller); - if (InlineSitesFromRemarks.empty()) - return std::make_unique(this, CB, None, ORE, - EmitRemarks); + Optional InlineRecommended; - std::string CallSiteLoc = getCallSiteLocation(CB.getDebugLoc()); - StringRef Callee = CB.getCalledFunction()->getName(); - std::string Combined = (Callee + CallSiteLoc).str(); - auto Iter = InlineSitesFromRemarks.find(Combined); + if (Scope == ReplayInlineScope::Module || + InlineCallersFromRemarks.count(CB.getFunction()->getName())) { + std::string CallSiteLoc = getCallSiteLocation(CB.getDebugLoc()); + StringRef Callee = CB.getCalledFunction()->getName(); + std::string Combined = (Callee + CallSiteLoc).str(); - Optional InlineRecommended = None; - if (Iter != InlineSitesFromRemarks.end()) { - InlineRecommended = llvm::InlineCost::getAlways("found in replay"); + auto Iter = InlineSitesFromRemarks.find(Combined); + if (Iter != InlineSitesFromRemarks.end()) { + InlineSitesFromRemarks[Combined] = true; + InlineRecommended = llvm::InlineCost::getAlways("previously inlined"); + } + } else if (Scope == ReplayInlineScope::Function) { + if (OriginalAdvisor) + return OriginalAdvisor->getAdvice(CB); + return {}; } return std::make_unique(this, CB, InlineRecommended, ORE, diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -97,7 +97,15 @@ "cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( "Optimization remarks file containing inline remarks to be replayed " - "by inlining from cgscc inline remarks."), + "by cgscc inlining."), + cl::Hidden); + +static cl::opt CGSCCInlineReplayScope( + "cgscc-inline-replay-scope", cl::init(""), + cl::value_desc(""), + cl::desc("Whether inline replay should be applied to the entire " + "Module(default) or just the Functions that are present as " + "callers in remarks during cgscc inlining."), cl::Hidden); static cl::opt InlineEnablePriorityOrder( @@ -664,7 +672,7 @@ if (!CGSCCInlineReplayFile.empty()) OwnedAdvisor = std::make_unique( M, FAM, M.getContext(), std::move(OwnedAdvisor), - CGSCCInlineReplayFile, + CGSCCInlineReplayFile, CGSCCInlineReplayScope, /*EmitRemarks=*/true); return *OwnedAdvisor; @@ -827,8 +835,9 @@ } auto Advice = Advisor.getAdvice(*CB, OnlyMandatory); + // Check whether we want to inline this callsite. - if (!Advice->isInliningRecommended()) { + if (!Advice || !Advice->isInliningRecommended()) { Advice->recordUnattemptedInlining(); continue; } @@ -1027,7 +1036,8 @@ PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M, ModuleAnalysisManager &MAM) { auto &IAA = MAM.getResult(M); - if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile)) { + if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile, + CGSCCInlineReplayScope)) { M.getContext().emitError( "Could not setup Inlining Advisor for the requested " "mode and/or options"); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -237,6 +237,14 @@ "by inlining from sample profile loader."), cl::Hidden); +static cl::opt ProfileInlineReplayScope( + "sample-profile-inline-replay-scope", cl::init(""), + cl::value_desc(""), + cl::desc("Whether inline replay should be applied to the entire " + "Module(default) or just the Functions that are present as " + "callers in remarks during sample profile inlining."), + cl::Hidden); + static cl::opt MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::ZeroOrMore, @@ -1272,12 +1280,14 @@ std::unique_ptr Advice = nullptr; if (ExternalInlineAdvisor) { Advice = ExternalInlineAdvisor->getAdvice(*Candidate.CallInstr); - if (!Advice->isInliningRecommended()) { - Advice->recordUnattemptedInlining(); - return InlineCost::getNever("not previously inlined"); + if (Advice) { + if (!Advice->isInliningRecommended()) { + Advice->recordUnattemptedInlining(); + return InlineCost::getNever("not previously inlined"); + } + Advice->recordInlining(); + return InlineCost::getAlways("previously inlined"); } - Advice->recordInlining(); - return InlineCost::getAlways("previously inlined"); } // Adjust threshold based on call site hotness, only do this for callsite @@ -1835,7 +1845,7 @@ if (FAM && !ProfileInlineReplayFile.empty()) { ExternalInlineAdvisor = std::make_unique( M, *FAM, Ctx, /*OriginalAdvisor=*/nullptr, ProfileInlineReplayFile, - /*EmitRemarks=*/false); + ProfileInlineReplayScope, /*EmitRemarks=*/false); if (!ExternalInlineAdvisor->areReplayRemarksLoaded()) ExternalInlineAdvisor.reset(); } diff --git a/llvm/test/Transforms/Inline/cgscc-inline-replay.ll b/llvm/test/Transforms/Inline/cgscc-inline-replay.ll --- a/llvm/test/Transforms/Inline/cgscc-inline-replay.ll +++ b/llvm/test/Transforms/Inline/cgscc-inline-replay.ll @@ -1,8 +1,32 @@ ;; Note that this needs new pass manager for now. Passing `-cgscc-inline-replay` to legacy pass manager is a no-op. +;; Check baseline inline decisions +; RUN: opt < %s -passes=inline -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=DEFAULT %s + ;; Check replay inline decisions -; RUN: opt < %s -passes=inline -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=DEFAULT %s -; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY %s +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY %s + +;; Check function scope inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=Function -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-FUNCTION %s + +;; Check behavior on non-existent replay file +; RUN: not opt < %s -passes=inline -cgscc-inline-replay=%S -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR %s + +;; Check scope inlining errors out on non inputs +; RUN: not opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=function -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-SCOPE %s + +; DEFAULT: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} +; DEFAULT: '_Z3sumii' inlined into 'main' with (cost={{[-0-9]+}} +; DEFAULT-NOT: '_Z3subii' inlined into 'main' + +; REPLAY: '_Z3sumii' inlined into 'main' with (cost=always) +; REPLAY: '_Z3subii' inlined into 'main' with (cost=always) + +; REPLAY-FUNCTION: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} +; REPLAY-FUNCTION: '_Z3sumii' inlined into 'main' with (cost=always) + +; REPLAY-ERROR: error: Could not open remarks file: Is a directory +; REPLAY-ERROR-SCOPE: error: Invalid argument for replay scope, must be 'Module' or 'Function' @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1 @@ -109,11 +133,3 @@ !24 = !DILexicalBlockFile(scope: !18, file: !1, discriminator: 6) !25 = !DILocation(line: 11, scope: !12) !26 = !DILocation(line: 12, scope: !12) - -; DEFAULT: '_Z3subii' inlined into '_Z3sumii' -; DEFAULT: '_Z3sumii' inlined into 'main' -; DEFAULT-NOT: '_Z3subii' inlined into 'main' - -; REPLAY: '_Z3sumii' inlined into 'main' -; REPLAY: '_Z3subii' inlined into 'main' -; REPLAY-NOT: '_Z3subii' inlined into '_Z3sumii' diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-strict.txt b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-strict.txt new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-strict.txt @@ -0,0 +1 @@ +remark: calls.cc:10:0: '_Z3sumii' inlined into 'main' to match profiling context with (cost=45, threshold=337) at callsite main:3:0.1; \ No newline at end of file diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt --- a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt +++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt @@ -1,2 +1,2 @@ remark: calls.cc:10:0: '_Z3sumii' inlined into 'main' to match profiling context with (cost=45, threshold=337) at callsite main:3:0.1; -remark: calls.cc:4:0: '_Z3subii' inlined into 'main' to match profiling context with (cost=-5, threshold=337) at callsite _Z3sumii:1:0 @ main:3:0.1; +remark: calls.cc:4:0: '_Z3subii' inlined into 'main' to match profiling context with (cost=-5, threshold=337) at callsite _Z3sumii:1:0 @ main:3:0.1; \ No newline at end of file diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-inline-all.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-inline-all.prof new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown-inline-all.prof @@ -0,0 +1,10 @@ +main:225715:0 + 2.1: 5553 + 3: 5391 + 3.1: _Z3sumii:50000 + 1: _Z3subii:50000 + 1: 0 + +_Z3sumii:6010:50000 + 1: _Z3subii:60000 + 1: 9 \ No newline at end of file diff --git a/llvm/test/Transforms/SampleProfile/inline-replay.ll b/llvm/test/Transforms/SampleProfile/inline-replay.ll --- a/llvm/test/Transforms/SampleProfile/inline-replay.ll +++ b/llvm/test/Transforms/SampleProfile/inline-replay.ll @@ -6,6 +6,39 @@ ;; Check replay inline decisions ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY %s +;; Check baseline inline decisions with "inline-topdown-inline-all.prof" which inlines all sites +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=DEFAULT-ALL %s + +;; Check function scope replay inline decisions with "inline-topdown-inline-all.prof" which only contains: '_Z3sumii' inlined into 'main' +;; 1. _Z3sumii is inlined into main, but all other inline candidates in main (e.g. _Z3subii) are not inlined +;; 2. Inline decisions made in other functions match default sample inlining, in this case _Z3subii is inlined into _Z3sumii +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-strict.txt -sample-profile-inline-replay-scope=Function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION %s + +;; Check behavior on non-existent replay file +; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR %s + +;; Check scope inlining errors out on non inputs +; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-SCOPE %s + +; DEFAULT: '_Z3sumii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} +; DEFAULT: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} +; DEFAULT-NOT: '_Z3subii' inlined into 'main' + +; REPLAY: '_Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY: '_Z3subii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-NOT: '_Z3subii' inlined into '_Z3sumii' + +; DEFAULT-ALL: '_Z3sumii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} +; DEFAULT-ALL: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} +; DEFAULT-ALL: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} + +; REPLAY-ALL-FUNCTION : _Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-ALL-FUNCTION-NOT: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} +; REPLAY-ALL-FUNCTION: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} + +; REPLAY-ERROR: error: Could not open remarks file: Is a directory +; REPLAY-ERROR-SCOPE: error: Invalid argument for replay scope, must be 'Module' or 'Function' + @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1 define i32 @_Z3sumii(i32 %x, i32 %y) #0 !dbg !6 { @@ -111,12 +144,3 @@ !24 = !DILexicalBlockFile(scope: !18, file: !1, discriminator: 6) !25 = !DILocation(line: 11, scope: !12) !26 = !DILocation(line: 12, scope: !12) - - -; DEFAULT: '_Z3sumii' inlined into 'main' -; DEFAULT: '_Z3subii' inlined into '_Z3sumii' -; DEFAULT-NOT: '_Z3subii' inlined into 'main' - -; REPLAY: '_Z3sumii' inlined into 'main' -; REPLAY: '_Z3subii' inlined into 'main' -; REPLAY-NOT: '_Z3subii' inlined into '_Z3sumii'