diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -38,8 +38,37 @@ /// training. enum class InliningAdvisorMode : int { Default, Release, Development }; -/// For Replay Inliner initialization -enum class ReplayInlineScope : int { Function, Module }; +struct CallSiteFormat { + enum class Format : int { + Line, + LineColumn, + LineDiscriminator, + LineColumnDiscriminator + }; + + bool OutputColumn() const { + return OutputFormat == Format::LineColumn || + OutputFormat == Format::LineColumnDiscriminator; + } + + bool OutputDiscriminator() const { + return OutputFormat == Format::LineDiscriminator || + OutputFormat == Format::LineColumnDiscriminator; + } + + Format OutputFormat; +}; + +/// Replay Inliner Setup +struct ReplayInlinerSettings { + enum class Scope : int { Function, Module }; + enum class Mode : int { Original, AlwaysInline, NeverInline }; + + StringRef ReplayFile; + Scope ReplayScope; + Mode ReplayMode; + CallSiteFormat ReplayFormat; +}; class InlineAdvisor; /// Capture state between an inlining decision having had been made, and @@ -234,7 +263,7 @@ return !PAC.preservedWhenStateless(); } bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, - StringRef ReplayFile, ReplayInlineScope ReplayScope); + ReplayInlinerSettings ReplaySettings); InlineAdvisor *getAdvisor() const { return Advisor.get(); } private: @@ -257,10 +286,11 @@ std::function GetDefaultAdvice); #endif -std::unique_ptr getReplayInlineAdvisor( - Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, - std::unique_ptr OriginalAdvisor, StringRef RemarksFile, - ReplayInlineScope Scope, bool EmitRemarks); +std::unique_ptr +getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, + LLVMContext &Context, + std::unique_ptr OriginalAdvisor, + ReplayInlinerSettings ReplaySettings, bool EmitRemarks); // Default (manual policy) decision making helper APIs. Shared with the legacy // pass manager inliner. @@ -288,7 +318,7 @@ const char *PassName = nullptr); /// get call site location as string -std::string getCallSiteLocation(DebugLoc DLoc); +std::string getCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format); /// Add location info to ORE message. void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc); diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h --- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h @@ -27,19 +27,20 @@ ReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr OriginalAdvisor, - StringRef RemarksFile, ReplayInlineScope Scope, + const ReplayInlinerSettings ReplaySettings, bool EmitRemarks); std::unique_ptr getAdviceImpl(CallBase &CB) override; bool areReplayRemarksLoaded() const { return HasReplayRemarks; } bool hasRemarksForFunction(Function &F) const { - return (Scope == ReplayInlineScope::Module) || + return (ReplaySettings.ReplayScope == + ReplayInlinerSettings::Scope::Module) || CallersToReplay.contains(F.getName()); } private: std::unique_ptr OriginalAdvisor; bool HasReplayRemarks = false; - ReplayInlineScope Scope; + ReplayInlinerSettings ReplaySettings; bool EmitRemarks = false; StringMap InlineSitesFromRemarks; diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -186,10 +186,9 @@ AnalysisKey InlineAdvisorAnalysis::Key; -bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params, - InliningAdvisorMode Mode, - StringRef ReplayFile, - ReplayInlineScope ReplayScope) { +bool InlineAdvisorAnalysis::Result::tryCreate( + InlineParams Params, InliningAdvisorMode Mode, + ReplayInlinerSettings ReplaySettings) { auto &FAM = MAM.getResult(M).getManager(); switch (Mode) { case InliningAdvisorMode::Default: @@ -197,10 +196,10 @@ Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params)); // Restrict replay to default advisor, ML advisors are stateful so // replay will need augmentations to interleave with them correctly. - if (!ReplayFile.empty()) { - Advisor = llvm::getReplayInlineAdvisor( - M, FAM, M.getContext(), std::move(Advisor), ReplayFile, ReplayScope, - /* EmitRemarks =*/true); + if (!ReplaySettings.ReplayFile.empty()) { + Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(), + std::move(Advisor), ReplaySettings, + /* EmitRemarks =*/true); } break; case InliningAdvisorMode::Development: @@ -419,7 +418,8 @@ return IC; } -std::string llvm::getCallSiteLocation(DebugLoc DLoc) { +std::string llvm::getCallSiteLocation(DebugLoc DLoc, + const CallSiteFormat &Format) { std::string Buffer; raw_string_ostream CallSiteLoc(Buffer); bool First = true; @@ -435,9 +435,10 @@ StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); if (Name.empty()) Name = DIL->getScope()->getSubprogram()->getName(); - CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset) << ":" - << llvm::utostr(DIL->getColumn()); - if (Discriminator) + CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset); + if (Format.OutputColumn()) + CallSiteLoc << ":" << llvm::utostr(DIL->getColumn()); + if (Format.OutputDiscriminator() && Discriminator) CallSiteLoc << "." << llvm::utostr(Discriminator); First = false; } diff --git a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp --- a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp @@ -25,12 +25,13 @@ ReplayInlineAdvisor::ReplayInlineAdvisor( Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, - std::unique_ptr OriginalAdvisor, StringRef RemarksFile, - ReplayInlineScope Scope, bool EmitRemarks) + std::unique_ptr OriginalAdvisor, + const ReplayInlinerSettings ReplaySettings, bool EmitRemarks) : InlineAdvisor(M, FAM), OriginalAdvisor(std::move(OriginalAdvisor)), - HasReplayRemarks(false), Scope(Scope), EmitRemarks(EmitRemarks) { + HasReplayRemarks(false), ReplaySettings(ReplaySettings), + EmitRemarks(EmitRemarks) { - auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(RemarksFile); + auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(ReplaySettings.ReplayFile); std::error_code EC = BufferOrErr.getError(); if (EC) { Context.emitError("Could not open remarks file: " + EC.message()); @@ -42,11 +43,19 @@ // main:3:1.1; // We use the callsite string after `at callsite` to replay inlining. line_iterator LineIt(*BufferOrErr.get(), /*SkipBlanks=*/true); + const std::string PositiveRemark = "' inlined into '"; + const std::string NegativeRemark = "' will not be inlined into '"; + for (; !LineIt.is_at_eof(); ++LineIt) { StringRef Line = *LineIt; auto Pair = Line.split(" at callsite "); - auto CalleeCaller = Pair.first.split("' inlined into '"); + bool IsPositiveRemark = true; + if (Pair.first.contains(NegativeRemark)) + IsPositiveRemark = false; + + auto CalleeCaller = + Pair.first.split(IsPositiveRemark ? PositiveRemark : NegativeRemark); StringRef Callee = CalleeCaller.first.rsplit(": '").second; StringRef Caller = CalleeCaller.second.rsplit("'").first; @@ -59,8 +68,8 @@ } std::string Combined = (Callee + CallSite).str(); - InlineSitesFromRemarks[Combined] = false; - if (Scope == ReplayInlineScope::Function) + InlineSitesFromRemarks[Combined] = IsPositiveRemark; + if (ReplaySettings.ReplayScope == ReplayInlinerSettings::Scope::Function) CallersToReplay.insert(Caller); } @@ -69,11 +78,10 @@ std::unique_ptr llvm::getReplayInlineAdvisor( Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, - std::unique_ptr OriginalAdvisor, StringRef RemarksFile, - ReplayInlineScope Scope, bool EmitRemarks) { + std::unique_ptr OriginalAdvisor, + ReplayInlinerSettings ReplaySettings, bool EmitRemarks) { auto Advisor = std::make_unique( - M, FAM, Context, std::move(OriginalAdvisor), RemarksFile, Scope, - EmitRemarks); + M, FAM, Context, std::move(OriginalAdvisor), ReplaySettings, EmitRemarks); if (!Advisor->areReplayRemarksLoaded()) Advisor.reset(); return Advisor; @@ -86,19 +94,40 @@ auto &ORE = FAM.getResult(Caller); Optional InlineRecommended; + bool ReplayMadeDecision = false; - if (Scope == ReplayInlineScope::Module || + // Decision could be made by replay system + if (ReplaySettings.ReplayScope == ReplayInlinerSettings::Scope::Module || CallersToReplay.count(CB.getFunction()->getName())) { - std::string CallSiteLoc = getCallSiteLocation(CB.getDebugLoc()); + std::string CallSiteLoc = + getCallSiteLocation(CB.getDebugLoc(), ReplaySettings.ReplayFormat); StringRef Callee = CB.getCalledFunction()->getName(); std::string Combined = (Callee + CallSiteLoc).str(); auto Iter = InlineSitesFromRemarks.find(Combined); if (Iter != InlineSitesFromRemarks.end()) { - InlineSitesFromRemarks[Combined] = true; - InlineRecommended = llvm::InlineCost::getAlways("previously inlined"); - } - } else if (Scope == ReplayInlineScope::Function) { + if (InlineSitesFromRemarks[Combined]) + InlineRecommended = llvm::InlineCost::getAlways("previously inlined"); + ReplayMadeDecision = true; + } else if (ReplaySettings.ReplayMode == + ReplayInlinerSettings::Mode::AlwaysInline) { + InlineRecommended = + llvm::InlineCost::getAlways("AlwaysInline replay mode"); + ReplayMadeDecision = true; + } else if (ReplaySettings.ReplayMode == + ReplayInlinerSettings::Mode::NeverInline) + ReplayMadeDecision = true; + + LLVM_DEBUG(dbgs() << "Replay Inliner: " + << (InlineRecommended ? "Inlined " : "Not Inlined ") + << Callee << " @ " << CallSiteLoc << "\n"); + } + + // No decisions from replay inliner, see if we defer to original inliner or + // return a non-decision + if (!ReplayMadeDecision && + ((ReplaySettings.ReplayScope == ReplayInlinerSettings::Scope::Function) || + (ReplaySettings.ReplayMode == ReplayInlinerSettings::Mode::Original))) { if (OriginalAdvisor) return OriginalAdvisor->getAdvice(CB); return {}; diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -100,18 +100,49 @@ "by cgscc inlining."), cl::Hidden); -static cl::opt CGSCCInlineReplayScope( - "cgscc-inline-replay-scope", cl::init(ReplayInlineScope::Function), - cl::values(clEnumValN(ReplayInlineScope::Function, "Function", +static cl::opt CGSCCInlineReplayScope( + "cgscc-inline-replay-scope", + cl::init(ReplayInlinerSettings::Scope::Function), + cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), - clEnumValN(ReplayInlineScope::Module, "Module", + clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden); +static cl::opt CGSCCInlineReplayMode( + "cgscc-inline-replay-mode", cl::init(ReplayInlinerSettings::Mode::Original), + cl::values( + clEnumValN( + ReplayInlinerSettings::Mode::Original, "Original", + "All decisions not in replay send to original advisor (default)"), + clEnumValN(ReplayInlinerSettings::Mode::AlwaysInline, "AlwaysInline", + "All decisions not in replay are inlined"), + clEnumValN(ReplayInlinerSettings::Mode::NeverInline, "NeverInline", + "All decisions not in replay are not inlined")), + cl::desc( + "How cgscc inline replay treats sites that don't come from the replay. " + "Original: defers to original advisor, AlwaysInline: inline all sites " + "not in replay, NeverInline: inline no sites not in replay"), + cl::Hidden); + +static cl::opt CGSCCInlineReplayFormat( + "cgscc-inline-replay-format", + cl::init(CallSiteFormat::Format::LineColumnDiscriminator), + cl::values( + clEnumValN(CallSiteFormat::Format::Line, "Line", ""), + clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", + ":"), + clEnumValN(CallSiteFormat::Format::LineDiscriminator, + "LineDiscriminator", "."), + clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, + "LineColumnDiscriminator", + ":. (default)")), + cl::desc("How cgscc inline replay file is formatted"), cl::Hidden); + static cl::opt InlineEnablePriorityOrder( "inline-enable-priority-order", cl::Hidden, cl::init(false), cl::desc("Enable the priority inline order for the inliner")); @@ -676,7 +707,10 @@ if (!CGSCCInlineReplayFile.empty()) OwnedAdvisor = getReplayInlineAdvisor( M, FAM, M.getContext(), std::move(OwnedAdvisor), - CGSCCInlineReplayFile, CGSCCInlineReplayScope, + ReplayInlinerSettings{CGSCCInlineReplayFile, + CGSCCInlineReplayScope, + CGSCCInlineReplayMode, + {CGSCCInlineReplayFormat}}, /*EmitRemarks=*/true); return *OwnedAdvisor; @@ -1040,8 +1074,11 @@ PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M, ModuleAnalysisManager &MAM) { auto &IAA = MAM.getResult(M); - if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile, - CGSCCInlineReplayScope)) { + if (!IAA.tryCreate(Params, Mode, + {CGSCCInlineReplayFile, + CGSCCInlineReplayScope, + CGSCCInlineReplayMode, + {CGSCCInlineReplayFormat}})) { M.getContext().emitError( "Could not setup Inlining Advisor for the requested " "mode and/or options"); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -237,18 +237,50 @@ "by inlining from sample profile loader."), cl::Hidden); -static cl::opt ProfileInlineReplayScope( - "sample-profile-inline-replay-scope", cl::init(ReplayInlineScope::Function), - cl::values(clEnumValN(ReplayInlineScope::Function, "Function", +static cl::opt ProfileInlineReplayScope( + "sample-profile-inline-replay-scope", + cl::init(ReplayInlinerSettings::Scope::Function), + cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), - clEnumValN(ReplayInlineScope::Module, "Module", + clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining."), cl::Hidden); +static cl::opt ProfileInlineReplayMode( + "sample-profile-inline-replay-mode", + cl::init(ReplayInlinerSettings::Mode::Original), + cl::values( + clEnumValN( + ReplayInlinerSettings::Mode::Original, "Original", + "All decisions not in replay send to original advisor (default)"), + clEnumValN(ReplayInlinerSettings::Mode::AlwaysInline, "AlwaysInline", + "All decisions not in replay are inlined"), + clEnumValN(ReplayInlinerSettings::Mode::NeverInline, "NeverInline", + "All decisions not in replay are not inlined")), + cl::desc("How sample profile inline replay treats sites that don't come " + "from the replay. Original: defers to original advisor, " + "AlwaysInline: inline all sites not in replay, NeverInline: " + "inline no sites not in replay"), + cl::Hidden); + +static cl::opt ProfileInlineReplayFormat( + "sample-profile-inline-replay-format", + cl::init(CallSiteFormat::Format::LineColumnDiscriminator), + cl::values( + clEnumValN(CallSiteFormat::Format::Line, "Line", ""), + clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", + ":"), + clEnumValN(CallSiteFormat::Format::LineDiscriminator, + "LineDiscriminator", "."), + clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, + "LineColumnDiscriminator", + ":. (default)")), + cl::desc("How sample profile inline replay file is formatted"), cl::Hidden); + static cl::opt MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::ZeroOrMore, @@ -1862,8 +1894,12 @@ if (FAM && !ProfileInlineReplayFile.empty()) { ExternalInlineAdvisor = std::make_unique( - M, *FAM, Ctx, /*OriginalAdvisor=*/nullptr, ProfileInlineReplayFile, - ProfileInlineReplayScope, /*EmitRemarks=*/false); + M, *FAM, Ctx, /*OriginalAdvisor=*/nullptr, + ReplayInlinerSettings{ProfileInlineReplayFile, + ProfileInlineReplayScope, + ProfileInlineReplayMode, + {ProfileInlineReplayFormat}}, + /*EmitRemarks=*/false); } // Apply tweaks if context-sensitive profile is available. diff --git a/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay-function.txt b/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay-function.txt new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay-function.txt @@ -0,0 +1 @@ +remark: calls.cc:4:0: '_Z3subii' will not be inlined into 'main' with (cost=-5, threshold=337) at callsite _Z3sumii:1:0 @ main:3:0.1; diff --git a/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay-line.txt b/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay-line.txt new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay-line.txt @@ -0,0 +1,2 @@ +remark: calls.cc:4: '_Z3subii' will not be inlined into '_Z3sumii' with (cost=Never) at callsite _Z3sumii:1; +remark: calls.cc:10: '_Z3sumii' inlined into 'main' with (cost=45, threshold=337) at callsite main:3; diff --git a/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay.txt b/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay.txt --- a/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay.txt +++ b/llvm/test/Transforms/Inline/Inputs/cgscc-inline-replay.txt @@ -1,2 +1,2 @@ +remark: calls.cc:4:0: '_Z3subii' will not be inlined into '_Z3sumii' with (cost=Never) at callsite _Z3sumii:1:0; remark: calls.cc:10:0: '_Z3sumii' inlined into 'main' with (cost=45, threshold=337) at callsite main:3:0.1; -remark: calls.cc:4:0: '_Z3subii' inlined into 'main' with (cost=-5, threshold=337) at callsite _Z3sumii:1:0 @ main:3:0.1; diff --git a/llvm/test/Transforms/Inline/cgscc-inline-replay.ll b/llvm/test/Transforms/Inline/cgscc-inline-replay.ll --- a/llvm/test/Transforms/Inline/cgscc-inline-replay.ll +++ b/llvm/test/Transforms/Inline/cgscc-inline-replay.ll @@ -3,11 +3,27 @@ ;; Check baseline inline decisions ; RUN: opt < %s -passes=inline -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=DEFAULT %s -;; Check module-scope replay inline decisions -; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=Module -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY %s +;; Check Module scope Original mode replay inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=Module -cgscc-inline-replay-mode=Original -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-ORIGINAL %s -;; Check function-scope inline replay decisions -; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=Function -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-FUNCTION %s +;; Check Module scope Original mode replay inline with 'Line' format decisions +;; The results are not different than REPLAY-MODULE-ORIGINAL, but the replay input only contains line numbers rather than line:column.discriminator +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay-line.txt -cgscc-inline-replay-scope=Module -cgscc-inline-replay-mode=Original -cgscc-inline-replay-format=Line -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-ORIGINAL %s + +;; Check Module scope AlwaysInline mode replay inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=Module -cgscc-inline-replay-mode=AlwaysInline -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-ALWAYS %s + +;; Check Module scope NeverInline mode replay inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=Module -cgscc-inline-replay-mode=NeverInline -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-NEVER %s + +;; Check Function scope Original mode replay inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay-function.txt -cgscc-inline-replay-scope=Function -cgscc-inline-replay-mode=Original -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-FUNCTION-ORIGINAL %s + +;; Check Function scope AlwaysInline mode replay inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay-function.txt -cgscc-inline-replay-scope=Function -cgscc-inline-replay-mode=AlwaysInline -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-FUNCTION-ALWAYS %s + +;; Check Function scope NeverInline mode replay inline decisions +; RUN: opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay-function.txt -cgscc-inline-replay-scope=Function -cgscc-inline-replay-mode=NeverInline -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-FUNCTION-NEVER %s ;; Check behavior on non-existent replay file ; RUN: not opt < %s -passes=inline -cgscc-inline-replay=%S -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR %s @@ -15,18 +31,38 @@ ;; Check scope inlining errors out on non inputs ; RUN: not opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-scope=function -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-SCOPE %s +;; Check mode inlining errors out on non inputs +; RUN: not opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-mode=original -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-MODE %s + +;; Check format inlining errors out on non inputs +; RUN: not opt < %s -passes=inline -cgscc-inline-replay=%S/Inputs/cgscc-inline-replay.txt -cgscc-inline-replay-format=line -pass-remarks=inline --disable-output 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-FORMAT %s + ; DEFAULT: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} ; DEFAULT: '_Z3sumii' inlined into 'main' with (cost={{[-0-9]+}} ; DEFAULT-NOT: '_Z3subii' inlined into 'main' -; REPLAY: '_Z3sumii' inlined into 'main' with (cost=always) -; REPLAY: '_Z3subii' inlined into 'main' with (cost=always) +; REPLAY-MODULE-ORIGINAL: '_Z3sumii' inlined into 'main' with (cost=always) +; REPLAY-MODULE-ORIGINAL: '_Z3subii' inlined into 'main' with (cost={{[-0-9]+}} + +; REPLAY-MODULE-ALWAYS: '_Z3sumii' inlined into 'main' with (cost=always) +; REPLAY-MODULE-ALWAYS: '_Z3subii' inlined into 'main' with (cost=always) + +; REPLAY-MODULE-NEVER: '_Z3sumii' inlined into 'main' with (cost=always) +; REPLAY-MODULE-NEVER-NOT: '_Z3subii' inlined into 'main' + +; REPLAY-FUNCTION-ORIGINAL: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} +; REPLAY-FUNCTION-ORIGINAL: '_Z3sumii' inlined into 'main' with (cost={{[-0-9]+}} + +; REPLAY-FUNCTION-ALWAYS: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} +; REPLAY-FUNCTION-ALWAYS: '_Z3sumii' inlined into 'main' with (cost=always) -; REPLAY-FUNCTION: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} -; REPLAY-FUNCTION: '_Z3sumii' inlined into 'main' with (cost=always) +; REPLAY-FUNCTION-NEVER: '_Z3subii' inlined into '_Z3sumii' with (cost={{[-0-9]+}} +; REPLAY-FUNCTION-NEVER-NOT: '_Z3sumii' inlined into 'main' ; REPLAY-ERROR: error: Could not open remarks file: ; REPLAY-ERROR-SCOPE: for the --cgscc-inline-replay-scope option: Cannot find option named 'function'! +; REPLAY-ERROR-MODE: for the --cgscc-inline-replay-mode option: Cannot find option named 'original'! +; REPLAY-ERROR-FORMAT: for the --cgscc-inline-replay-format option: Cannot find option named 'line'! @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1 diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-function-scope.txt b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-function.txt rename from llvm/test/Transforms/SampleProfile/Inputs/inline-replay-function-scope.txt rename to llvm/test/Transforms/SampleProfile/Inputs/inline-replay-function.txt diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-line.txt b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-line.txt new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay-line.txt @@ -0,0 +1,2 @@ +remark: calls.cc:4: '_Z3subii' will not be inlined into '_Z3sumii' to match profiling context with (cost=Never) at callsite _Z3sumii:1; +remark: calls.cc:10: '_Z3sumii' inlined into 'main' to match profiling context with (cost=45, threshold=337) at callsite main:3; \ No newline at end of file diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt --- a/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt +++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-replay.txt @@ -1,2 +1,2 @@ -remark: calls.cc:10:0: '_Z3sumii' inlined into 'main' to match profiling context with (cost=45, threshold=337) at callsite main:3:0.1; -remark: calls.cc:4:0: '_Z3subii' inlined into 'main' to match profiling context with (cost=-5, threshold=337) at callsite _Z3sumii:1:0 @ main:3:0.1; \ No newline at end of file +remark: calls.cc:4:0: '_Z3subii' will not be inlined into '_Z3sumii' to match profiling context with (cost=Never) at callsite _Z3sumii:1:0; +remark: calls.cc:10:0: '_Z3sumii' inlined into 'main' to match profiling context with (cost=45, threshold=337) at callsite main:3:0.1; \ No newline at end of file diff --git a/llvm/test/Transforms/SampleProfile/inline-replay.ll b/llvm/test/Transforms/SampleProfile/inline-replay.ll --- a/llvm/test/Transforms/SampleProfile/inline-replay.ll +++ b/llvm/test/Transforms/SampleProfile/inline-replay.ll @@ -3,8 +3,18 @@ ;; Check baseline inline decisions ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=DEFAULT %s -;; Check replay inline decisions -; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY %s +;; Check Module scope Original mode replay inline decisions +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-inline-replay-mode=Original -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-ORIGINAL %s + +;; Check Module scope Original mode replay inline with 'Line' format decisions +;; The results are not different than REPLAY-MODULE-ORIGINAL, but the replay input only contains line numbers rather than line:column.discriminator +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-line.txt -sample-profile-inline-replay-scope=Module -sample-profile-inline-replay-mode=Original -sample-profile-inline-replay-format=Line -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-ORIGINAL %s + +;; Check Module scope AlwaysInline mode replay inline decisions +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-inline-replay-mode=AlwaysInline -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-ALWAYS %s + +;; Check Module scope NeverInline mode replay inline decisions +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=Module -sample-profile-inline-replay-mode=NeverInline -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-MODULE-NEVER %s ;; Check baseline inline decisions with "inline-topdown-inline-all.prof" which inlines all sites ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=DEFAULT-ALL %s @@ -12,7 +22,13 @@ ;; Check function scope replay inline decisions with "inline-topdown-inline-all.prof" and "inline-topdown-function-scope.txt" which only contains: '_Z3sumii' inlined into 'main' ;; 1. _Z3sumii is inlined into main, but all other inline candidates in main (e.g. _Z3subii) are not inlined ;; 2. Inline decisions made in other functions match default sample inlining, in this case _Z3subii is inlined into _Z3sumii -; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-function-scope.txt -sample-profile-inline-replay-scope=Function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION %s +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-function.txt -sample-profile-inline-replay-scope=Function -sample-profile-inline-replay-mode=NeverInline -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION-NEVER %s + +;; Function scope Original mode +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-function.txt -sample-profile-inline-replay-scope=Function -sample-profile-inline-replay-mode=Original -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION-ORIGINAL %s + +;; Function scope AlwaysInline mode +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown-inline-all.prof -sample-profile-inline-replay=%S/Inputs/inline-replay-function.txt -sample-profile-inline-replay-scope=Function -sample-profile-inline-replay-mode=AlwaysInline -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ALL-FUNCTION-ALWAYS %s ;; Check behavior on non-existent replay file ; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR %s @@ -20,24 +36,45 @@ ;; Check scope inlining errors out on non inputs ; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-scope=function -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-SCOPE %s +;; Check mode inlining errors out on non inputs +; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-mode=original -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-MODE %s + +;; Check format inlining errors out on non inputs +; RUN: not opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-inline-replay=%S/Inputs/inline-replay.txt -sample-profile-inline-replay-format=line -sample-profile-merge-inlinee -sample-profile-top-down-load -pass-remarks=inline -S 2>&1 | FileCheck -check-prefix=REPLAY-ERROR-FORMAT %s + ; DEFAULT: '_Z3sumii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} ; DEFAULT: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} ; DEFAULT-NOT: '_Z3subii' inlined into 'main' -; REPLAY: '_Z3sumii' inlined into 'main' to match profiling context with (cost=always) -; REPLAY: '_Z3subii' inlined into 'main' to match profiling context with (cost=always) -; REPLAY-NOT: '_Z3subii' inlined into '_Z3sumii' +; REPLAY-MODULE-ORIGINAL: '_Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-MODULE-ORIGINAL: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} + +; REPLAY-MODULE-ALWAYS: '_Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-MODULE-ALWAYS: '_Z3subii' inlined into 'main' to match profiling context with (cost=always) + +; REPLAY-MODULE-NEVER: '_Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-MODULE-NEVER-NOT: '_Z3subii' inlined into 'main' ; DEFAULT-ALL: '_Z3sumii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} ; DEFAULT-ALL: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} ; DEFAULT-ALL: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} -; REPLAY-ALL-FUNCTION : _Z3sumii' inlined into 'main' to match profiling context with (cost=always) -; REPLAY-ALL-FUNCTION-NOT: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} -; REPLAY-ALL-FUNCTION: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} +; REPLAY-ALL-FUNCTION-NEVER: _Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-ALL-FUNCTION-NEVER-NOT: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} +; REPLAY-ALL-FUNCTION-NEVER: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} + +; REPLAY-ALL-FUNCTION-ORIGINAL: _Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-ALL-FUNCTION-ORIGINAL: '_Z3subii' inlined into 'main' to match profiling context with (cost={{[-0-9]+}} +; REPLAY-ALL-FUNCTION-ORIGINAL: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} + +; REPLAY-ALL-FUNCTION-ALWAYS: _Z3sumii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-ALL-FUNCTION-ALWAYS: '_Z3subii' inlined into 'main' to match profiling context with (cost=always) +; REPLAY-ALL-FUNCTION-ALWAYS: '_Z3subii' inlined into '_Z3sumii' to match profiling context with (cost={{[-0-9]+}} ; REPLAY-ERROR: error: Could not open remarks file: ; REPLAY-ERROR-SCOPE: for the --sample-profile-inline-replay-scope option: Cannot find option named 'function'! +; REPLAY-ERROR-MODE: for the --sample-profile-inline-replay-mode option: Cannot find option named 'original'! +; REPLAY-ERROR-FORMAT: for the --sample-profile-inline-replay-format option: Cannot find option named 'line'! @.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1