diff --git a/clang/test/Frontend/optimization-remark-line-directive.c b/clang/test/Frontend/optimization-remark-line-directive.c --- a/clang/test/Frontend/optimization-remark-line-directive.c +++ b/clang/test/Frontend/optimization-remark-line-directive.c @@ -6,7 +6,7 @@ // The new PM inliner is not added to the default pipeline at O0, so we add // some optimizations to trigger it. -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0 +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify int foo(int x, int y) __attribute__((always_inline)); int foo(int x, int y) { return x + y; } diff --git a/clang/test/Frontend/optimization-remark-new-pm.c b/clang/test/Frontend/optimization-remark-new-pm.c --- a/clang/test/Frontend/optimization-remark-new-pm.c +++ b/clang/test/Frontend/optimization-remark-new-pm.c @@ -1,8 +1,8 @@ // Verify that remarks for the inliner appear. The remarks under the new PM will // be slightly different than those emitted by the legacy PM. The new PM inliner // also doesnot appear to be added at O0, so we test at O1. -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0 -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -mllvm -mandatory-inlining-first=0 +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify int foo(int x, int y) __attribute__((always_inline)); int foo(int x, int y) { return x + y; } diff --git a/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c b/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c --- a/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c +++ b/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c @@ -18,46 +18,46 @@ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ -// RUN: -fdiagnostics-show-hotness -verify -mllvm -mandatory-inlining-first=0 +// RUN: -fdiagnostics-show-hotness -verify // The clang version of the previous test. // RUN: %clang -target x86_64-apple-macosx10.9 %s -c -emit-llvm -o /dev/null \ // RUN: -fprofile-instr-use=%t.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ -// RUN: -fdiagnostics-show-hotness -Xclang -verify -mllvm -mandatory-inlining-first=0 +// RUN: -fdiagnostics-show-hotness -Xclang -verify // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-sample-use=%t-sample.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ // RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 \ -// RUN: -verify -mllvm -mandatory-inlining-first=0 +// RUN: -verify // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ -// RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify -mllvm -mandatory-inlining-first=0 +// RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ -// RUN: -fexperimental-new-pass-manager -O1 -mllvm -mandatory-inlining-first=0 \ +// RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline 2>&1 | FileCheck -check-prefix=HOTNESS_OFF %s // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ -// RUN: -fexperimental-new-pass-manager -O1 -mllvm -mandatory-inlining-first=0 \ +// RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rno-pass-with-hotness 2>&1 | FileCheck \ // RUN: -check-prefix=HOTNESS_OFF %s // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ -// RUN: -Rpass-analysis=inline -fdiagnostics-show-hotness -mllvm -mandatory-inlining-first=0 \ +// RUN: -Rpass-analysis=inline -fdiagnostics-show-hotness \ // RUN: -fdiagnostics-hotness-threshold=100 2>&1 \ // RUN: | FileCheck -allow-empty -check-prefix=THRESHOLD %s // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ -// RUN: -Rpass=inline -Rpass-analysis=inline -mllvm -mandatory-inlining-first=0 \ +// RUN: -Rpass=inline -Rpass-analysis=inline \ // RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 2>&1 \ // RUN: | FileCheck -check-prefix=NO_PGO %s diff --git a/clang/test/Frontend/optimization-remark.c b/clang/test/Frontend/optimization-remark.c --- a/clang/test/Frontend/optimization-remark.c +++ b/clang/test/Frontend/optimization-remark.c @@ -7,30 +7,30 @@ // The inliner for the new PM does not seem to be enabled at O0, but we still // get the same remarks with at least O1. The remarks are also slightly // different and located in another test file. -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0 -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -mllvm -mandatory-inlining-first=0 -// RUN: %clang_cc1 %s -Rpass=inline -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -verify +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify +// RUN: %clang_cc1 %s -Rpass=inline -emit-llvm -o - 2>/dev/null | FileCheck %s // // Check that we can override -Rpass= with -Rno-pass. -// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS // // The inliner for the new PM does not seem to be enabled at O0, but we still // get the same remarks with at least O1. -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS // // Check that -w doesn't disable remarks. -// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS // // FIXME: -Reverything should imply -Rpass=.*. -// RUN: %clang_cc1 %s -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Reverything -emit-llvm -o - 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS // // FIXME: -Rpass should either imply -Rpass=.* or should be rejected. -// RUN: %clang_cc1 %s -Rpass -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Rpass -emit-llvm -o - 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS // CHECK-REMARKS: remark: // CHECK-NO-REMARKS-NOT: remark: diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -22,11 +22,9 @@ class Module; class OptimizationRemarkEmitter; -/// There are 4 scenarios we can use the InlineAdvisor: +/// There are 3 scenarios we can use the InlineAdvisor: /// - Default - use manual heuristics. /// -/// - MandatoryOnly - only mandatory inlinings (i.e. AlwaysInline). -/// /// - Release mode, the expected mode for production, day to day deployments. /// In this mode, when building the compiler, we also compile a pre-trained ML /// model to native code, and link it as a static library. This mode has low @@ -39,7 +37,6 @@ /// training. enum class InliningAdvisorMode : int { Default, - MandatoryOnly, Release, Development }; @@ -148,9 +145,12 @@ /// Get an InlineAdvice containing a recommendation on whether to /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to - /// be up-to-date wrt previous inlining decisions. + /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates + /// only mandatory (always-inline) call sites should be recommended - this + /// allows the InlineAdvisor track such inlininings. /// Returns an InlineAdvice with the inlining recommendation. - virtual std::unique_ptr getAdvice(CallBase &CB) = 0; + std::unique_ptr getAdvice(CallBase &CB, + bool MandatoryOnly = false); /// This must be called when the Inliner pass is entered, to allow the /// InlineAdvisor update internal state, as result of function passes run @@ -164,6 +164,9 @@ protected: InlineAdvisor(FunctionAnalysisManager &FAM) : FAM(FAM) {} + virtual std::unique_ptr getAdviceImpl(CallBase &CB) = 0; + virtual std::unique_ptr getMandatoryAdvice(CallBase &CB, + bool Advice); FunctionAnalysisManager &FAM; @@ -180,6 +183,14 @@ return DeletedFunctions.count(F); } + enum class MandatoryInliningKind { NotMandatory, Always, Never }; + + static MandatoryInliningKind getMandatoryKind(CallBase &CB, + FunctionAnalysisManager &FAM, + OptimizationRemarkEmitter &ORE); + + OptimizationRemarkEmitter &getCallerORE(CallBase &CB); + private: friend class InlineAdvice; void markFunctionAsDeleted(Function *F); @@ -195,27 +206,13 @@ : InlineAdvisor(FAM), Params(Params) {} private: - std::unique_ptr getAdvice(CallBase &CB) override; + std::unique_ptr getAdviceImpl(CallBase &CB) override; void onPassExit() override { freeDeletedFunctions(); } InlineParams Params; }; -/// Advisor recommending only mandatory (AlwaysInline) cases. -class MandatoryInlineAdvisor final : public InlineAdvisor { - std::unique_ptr getAdvice(CallBase &CB) override; - -public: - MandatoryInlineAdvisor(FunctionAnalysisManager &FAM) : InlineAdvisor(FAM) {} - - enum class MandatoryInliningKind { NotMandatory, Always, Never }; - - static MandatoryInliningKind getMandatoryKind(CallBase &CB, - FunctionAnalysisManager &FAM, - OptimizationRemarkEmitter &ORE); -}; - /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor /// needs to capture state right before inlining commences over a module. class InlineAdvisorAnalysis : public AnalysisInfoMixin { diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h --- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h @@ -31,8 +31,6 @@ void onPassEntry() override; - std::unique_ptr getAdvice(CallBase &CB) override; - int64_t getIRSize(const Function &F) const { return F.getInstructionCount(); } void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted); @@ -42,8 +40,12 @@ const MLModelRunner &getModelRunner() const { return *ModelRunner.get(); } protected: - virtual std::unique_ptr - getMandatoryAdvice(CallBase &CB, OptimizationRemarkEmitter &ORE); + std::unique_ptr getAdviceImpl(CallBase &CB) override; + + std::unique_ptr getMandatoryAdvice(CallBase &CB, + bool Advice) override; + + virtual std::unique_ptr getMandatoryAdviceImpl(CallBase &CB); virtual std::unique_ptr getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE); diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h --- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h +++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h @@ -26,7 +26,7 @@ public: ReplayInlineAdvisor(FunctionAnalysisManager &FAM, LLVMContext &Context, StringRef RemarksFile, bool EmitRemarks); - std::unique_ptr getAdvice(CallBase &CB) override; + std::unique_ptr getAdviceImpl(CallBase &CB) override; bool areReplayRemarksLoaded() const { return HasReplayRemarks; } private: diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -353,8 +353,7 @@ /// Construct the module pipeline that performs inlining as well as /// the inlining-driven cleanups. ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, - ThinOrFullLTOPhase Phase, - bool MandatoryOnly); + ThinOrFullLTOPhase Phase); /// Construct the core LLVM module optimization pipeline. /// diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -96,10 +96,9 @@ /// passes be composed to achieve the same end result. class InlinerPass : public PassInfoMixin { public: - InlinerPass() = default; + InlinerPass(bool OnlyMandatory = false) : OnlyMandatory(OnlyMandatory) {} ~InlinerPass(); - InlinerPass(InlinerPass &&Arg) - : ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {} + InlinerPass(InlinerPass &&Arg) = default; PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR); @@ -108,7 +107,8 @@ InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM, FunctionAnalysisManager &FAM, Module &M); std::unique_ptr ImportedFunctionsStats; - Optional OwnedDefaultAdvisor; + std::unique_ptr OwnedAdvisor; + const bool OnlyMandatory; }; /// Module pass, wrapping the inliner pass. This works in conjunction with the @@ -121,6 +121,7 @@ public: ModuleInlinerWrapperPass( InlineParams Params = getInlineParams(), bool Debugging = false, + bool MandatoryFirst = true, InliningAdvisorMode Mode = InliningAdvisorMode::Default, unsigned MaxDevirtIterations = 0); ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default; diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp --- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp +++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp @@ -163,8 +163,6 @@ FAM.invalidate(*F); } - std::unique_ptr - getMandatoryAdvice(CallBase &CB, OptimizationRemarkEmitter &ORE) override; std::unique_ptr getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE) override; @@ -172,6 +170,7 @@ private: bool isLogging() const { return !!Logger; } + std::unique_ptr getMandatoryAdviceImpl(CallBase &CB) override; std::function GetDefaultAdvice; const bool IsDoingInference; @@ -416,14 +415,11 @@ } std::unique_ptr -DevelopmentModeMLInlineAdvisor::getMandatoryAdvice( - CallBase &CB, OptimizationRemarkEmitter &ORE) { - if (!isLogging()) - return MLInlineAdvisor::getMandatoryAdvice(CB, ORE); - +DevelopmentModeMLInlineAdvisor::getMandatoryAdviceImpl(CallBase &CB) { return std::make_unique( /*Advisor=*/this, - /*CB=*/CB, /*ORE=*/ORE, /*Recommendation=*/true, /*Logger=*/*Logger, + /*CB=*/CB, /*ORE=*/getCallerORE(CB), /*Recommendation=*/true, + /*Logger=*/*Logger, /*CallerSizeEstimateBefore=*/getNativeSizeEstimate(*CB.getCaller()), /*CalleeSizeEstimateBefore=*/ getNativeSizeEstimate(*CB.getCalledFunction()), diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -103,7 +103,8 @@ Params.EnableDeferral.getValueOr(false)); } -std::unique_ptr DefaultInlineAdvisor::getAdvice(CallBase &CB) { +std::unique_ptr +DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) { auto OIC = getDefaultInlineAdvice(CB, FAM, Params); return std::make_unique( this, CB, OIC, @@ -144,9 +145,6 @@ case InliningAdvisorMode::Default: Advisor.reset(new DefaultInlineAdvisor(FAM, Params)); break; - case InliningAdvisorMode::MandatoryOnly: - Advisor.reset(new MandatoryInlineAdvisor(FAM)); - break; case InliningAdvisorMode::Development: #ifdef LLVM_HAVE_TF_API Advisor = @@ -430,21 +428,14 @@ }); } -std::unique_ptr MandatoryInlineAdvisor::getAdvice(CallBase &CB) { - auto &Caller = *CB.getCaller(); - auto &Callee = *CB.getCalledFunction(); - auto &ORE = FAM.getResult(Caller); - - bool Advice = MandatoryInliningKind::Always == - MandatoryInlineAdvisor::getMandatoryKind(CB, FAM, ORE) && - &Caller != &Callee; - return std::make_unique(this, CB, ORE, Advice); +std::unique_ptr InlineAdvisor::getMandatoryAdvice(CallBase &CB, + bool Advice) { + return std::make_unique(this, CB, getCallerORE(CB), Advice); } -MandatoryInlineAdvisor::MandatoryInliningKind -MandatoryInlineAdvisor::getMandatoryKind(CallBase &CB, - FunctionAnalysisManager &FAM, - OptimizationRemarkEmitter &ORE) { +InlineAdvisor::MandatoryInliningKind +InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, + OptimizationRemarkEmitter &ORE) { auto &Callee = *CB.getCalledFunction(); auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { @@ -464,3 +455,17 @@ } return MandatoryInliningKind::NotMandatory; } + +std::unique_ptr InlineAdvisor::getAdvice(CallBase &CB, + bool MandatoryOnly) { + if (!MandatoryOnly) + return getAdviceImpl(CB); + bool Advice = CB.getCaller() != CB.getCalledFunction() && + MandatoryInliningKind::Always == + getMandatoryKind(CB, FAM, getCallerORE(CB)); + return getMandatoryAdvice(CB, Advice); +} + +OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { + return FAM.getResult(*CB.getCaller()); +} diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp --- a/llvm/lib/Analysis/MLInlineAdvisor.cpp +++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp @@ -168,7 +168,7 @@ return Ret; } -std::unique_ptr MLInlineAdvisor::getAdvice(CallBase &CB) { +std::unique_ptr MLInlineAdvisor::getAdviceImpl(CallBase &CB) { auto &Caller = *CB.getCaller(); auto &Callee = *CB.getCalledFunction(); @@ -178,17 +178,17 @@ auto &TIR = FAM.getResult(Callee); auto &ORE = FAM.getResult(Caller); - auto MandatoryKind = MandatoryInlineAdvisor::getMandatoryKind(CB, FAM, ORE); + auto MandatoryKind = InlineAdvisor::getMandatoryKind(CB, FAM, ORE); // If this is a "never inline" case, there won't be any changes to internal // state we need to track, so we can just return the base InlineAdvice, which // will do nothing interesting. // Same thing if this is a recursive case. - if (MandatoryKind == MandatoryInlineAdvisor::MandatoryInliningKind::Never || + if (MandatoryKind == InlineAdvisor::MandatoryInliningKind::Never || &Caller == &Callee) - return std::make_unique(this, CB, ORE, false); + return getMandatoryAdvice(CB, false); bool Mandatory = - MandatoryKind == MandatoryInlineAdvisor::MandatoryInliningKind::Always; + MandatoryKind == InlineAdvisor::MandatoryInliningKind::Always; // If we need to stop, we won't want to track anymore any state changes, so // we just return the base InlineAdvice, which acts as a noop. @@ -214,7 +214,7 @@ } if (Mandatory) - return getMandatoryAdvice(CB, ORE); + return getMandatoryAdvice(CB, true); auto NrCtantParams = 0; for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) { @@ -249,10 +249,22 @@ return std::make_unique(this, CB, ORE, ModelRunner->run()); } +std::unique_ptr MLInlineAdvisor::getMandatoryAdvice(CallBase &CB, + bool Advice) { + // Make sure we track inlinings in all cases - mandatory or not. + if (Advice && !ForceStop) + return getMandatoryAdviceImpl(CB); + + // If this is a "never inline" case, there won't be any changes to internal + // state we need to track, so we can just return the base InlineAdvice, which + // will do nothing interesting. + // Same if we are forced to stop - we don't track anymore. + return std::make_unique(this, CB, getCallerORE(CB), Advice); +} + std::unique_ptr -MLInlineAdvisor::getMandatoryAdvice(CallBase &CB, - OptimizationRemarkEmitter &ORE) { - return std::make_unique(this, CB, ORE, true); +MLInlineAdvisor::getMandatoryAdviceImpl(CallBase &CB) { + return std::make_unique(this, CB, getCallerORE(CB), true); } void MLInlineAdvice::reportContextForRemark( diff --git a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp --- a/llvm/lib/Analysis/ReplayInlineAdvisor.cpp +++ b/llvm/lib/Analysis/ReplayInlineAdvisor.cpp @@ -56,7 +56,7 @@ HasReplayRemarks = true; } -std::unique_ptr ReplayInlineAdvisor::getAdvice(CallBase &CB) { +std::unique_ptr ReplayInlineAdvisor::getAdviceImpl(CallBase &CB) { assert(HasReplayRemarks); Function &Caller = *CB.getCaller(); diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -926,8 +926,9 @@ return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel()); } -ModuleInlinerWrapperPass PassBuilder::buildInlinerPipeline( - OptimizationLevel Level, ThinOrFullLTOPhase Phase, bool MandatoryOnly) { +ModuleInlinerWrapperPass +PassBuilder::buildInlinerPipeline(OptimizationLevel Level, + ThinOrFullLTOPhase Phase) { InlineParams IP = getInlineParamsFromOptLevel(Level); if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) @@ -936,13 +937,9 @@ if (PGOOpt) IP.EnableDeferral = EnablePGOInlineDeferral; - ModuleInlinerWrapperPass MIWP( - IP, DebugLogging, - (MandatoryOnly ? InliningAdvisorMode::MandatoryOnly : UseInlineAdvisor), - MaxDevirtIterations); - - if (MandatoryOnly) - return MIWP; + ModuleInlinerWrapperPass MIWP(IP, DebugLogging, + PerformMandatoryInliningsFirst, + UseInlineAdvisor, MaxDevirtIterations); // Require the GlobalsAA analysis for the module so we can query it within // the CGSCC pipeline. @@ -1143,9 +1140,7 @@ if (EnableSyntheticCounts && !PGOOpt) MPM.addPass(SyntheticCountsPropagation()); - if (PerformMandatoryInliningsFirst) - MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/true)); - MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/false)); + MPM.addPass(buildInlinerPipeline(Level, Phase)); if (EnableMemProfiler && Phase != ThinOrFullLTOPhase::ThinLTOPreLink) { MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); @@ -1639,9 +1634,6 @@ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); - MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level), - DebugLogging, - InliningAdvisorMode::MandatoryOnly)); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -62,10 +62,10 @@ MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true)) MODULE_PASS("inferattrs", InferFunctionAttrsPass()) MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass()) -MODULE_PASS("always-inliner-wrapper", ModuleInlinerWrapperPass( +MODULE_PASS("inliner-wrapper-no-mandatory-first", ModuleInlinerWrapperPass( getInlineParams(), DebugLogging, - InliningAdvisorMode::MandatoryOnly)) + false)) MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass()) MODULE_PASS("instrorderfile", InstrOrderFilePass()) MODULE_PASS("instrprof", InstrProfiling()) @@ -98,8 +98,7 @@ MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass()) MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("scc-oz-module-inliner", - buildInlinerPipeline(OptimizationLevel::Oz, ThinOrFullLTOPhase::None, - /*MandatoryOnly=*/false)) + buildInlinerPipeline(OptimizationLevel::Oz, ThinOrFullLTOPhase::None)) MODULE_PASS("loop-extract-single", LoopExtractorPass(1)) MODULE_PASS("strip", StripSymbolsPass()) MODULE_PASS("strip-dead-debug-info", StripDeadDebugInfoPass()) diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -668,8 +668,9 @@ // duration of the inliner pass, and thus the lifetime of the owned advisor. // The one we would get from the MAM can be invalidated as a result of the // inliner's activity. - OwnedDefaultAdvisor.emplace(FAM, getInlineParams()); - return *OwnedDefaultAdvisor; + OwnedAdvisor = + std::make_unique(FAM, getInlineParams()); + return *OwnedAdvisor; } assert(IAA->getAdvisor() && "Expected a present InlineAdvisorAnalysis also have an " @@ -832,7 +833,7 @@ continue; } - auto Advice = Advisor.getAdvice(*CB); + auto Advice = Advisor.getAdvice(*CB, OnlyMandatory); // Check whether we want to inline this callsite. if (!Advice->isInliningRecommended()) { Advice->recordUnattemptedInlining(); @@ -1017,6 +1018,7 @@ ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params, bool Debugging, + bool MandatoryFirst, InliningAdvisorMode Mode, unsigned MaxDevirtIterations) : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations), @@ -1026,6 +1028,8 @@ // into the callers so that our optimizations can reflect that. // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO // because it makes profile annotation in the backend inaccurate. + if (MandatoryFirst) + PM.addPass(InlinerPass(/*OnlyMandatory*/ true)); PM.addPass(InlinerPass()); } diff --git a/llvm/test/Other/new-pm-module-inliner-wrapper.ll b/llvm/test/Other/new-pm-module-inliner-wrapper.ll deleted file mode 100644 --- a/llvm/test/Other/new-pm-module-inliner-wrapper.ll +++ /dev/null @@ -1,7 +0,0 @@ -; RUN: opt -disable-verify -debug-pass-manager -passes='always-inliner-wrapper' -S %s 2>&1 | FileCheck %s - -; CHECK: Running pass: InlinerPass - -define void @foo() { - ret void -} diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll --- a/llvm/test/Transforms/Inline/inline_stats.ll +++ b/llvm/test/Transforms/Inline/inline_stats.ll @@ -6,11 +6,11 @@ ; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK ; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK -; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK -; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK +; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK +; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK -; RUN: opt -S -passes=always-inliner-wrapper,inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER -; RUN: opt -S -passes=always-inliner-wrapper,inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes=WRAPPER-VERBOSE,WRAPPER +; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefix=MANDATORY-FIRST +; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefix=MANDATORY-FIRST ; CHECK: ------- Dumping inliner stats for [] ------- ; CHECK-BASIC-NOT: -- List of inlined functions: @@ -30,15 +30,20 @@ ; CHECK: non-imported functions inlined anywhere: 1 [33.33% of non-imported functions] ; CHECK: non-imported functions inlined into importing module: 1 [33.33% of non-imported functions] -; WRAPPER-VERBOSE: -- List of inlined functions: -; WRAPPER-VERBOSE: Inlined imported function [external5]: #inlines = 1, #inlines_to_importing_module = 1 -; WRAPPER: -- Summary: -; WRAPPER: All functions: 10, imported functions: 7 -; WRAPPER: inlined functions: 1 [10% of all functions] -; WRAPPER: imported functions inlined anywhere: 1 [14.29% of imported functions] -; WRAPPER: imported functions inlined into importing module: 1 [14.29% of imported functions], remaining: 6 [85.71% of imported functions] -; WRAPPER: non-imported functions inlined anywhere: 0 [0% of non-imported functions] -; WRAPPER: non-imported functions inlined into importing module: 0 [0% of non-imported functions] +; MANDATORY-FIRST: - Summary: +; MANDATORY-FIRST: All functions: 10, imported functions: 7 +; MANDATORY-FIRST: inlined functions: 4 [40% of all functions] +; MANDATORY-FIRST: imported functions inlined anywhere: 3 [42.86% of imported functions] +; MANDATORY-FIRST: imported functions inlined into importing module: 2 [28.57% of imported functions], remaining: 5 [71.43% of imported functions] +; MANDATORY-FIRST: non-imported functions inlined anywhere: 1 [33.33% of non-imported functions] +; MANDATORY-FIRST: non-imported functions inlined into importing module: 1 [33.33% of non-imported functions] +; MANDATORY-FIRST: -- Summary: +; MANDATORY-FIRST: All functions: 10, imported functions: 7 +; MANDATORY-FIRST: inlined functions: 1 [10% of all functions] +; MANDATORY-FIRST: imported functions inlined anywhere: 1 [14.29% of imported functions] +; MANDATORY-FIRST: imported functions inlined into importing module: 1 [14.29% of imported functions], remaining: 6 [85.71% of imported functions] +; MANDATORY-FIRST: non-imported functions inlined anywhere: 0 [0% of non-imported functions] +; MANDATORY-FIRST: non-imported functions inlined into importing module: 0 [0% of non-imported functions] define void @internal() { call fastcc void @external1() diff --git a/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll b/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll --- a/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll +++ b/llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll @@ -4,7 +4,7 @@ ; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \ ; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \ ; RUN: | FileCheck %s -; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \ +; RUN: opt < %s -passes=inliner-wrapper-no-mandatory-first -pass-remarks=inline -pass-remarks-missed=inline \ ; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \ ; RUN: | FileCheck %s diff --git a/llvm/test/Transforms/Inline/optimization-remarks.ll b/llvm/test/Transforms/Inline/optimization-remarks.ll --- a/llvm/test/Transforms/Inline/optimization-remarks.ll +++ b/llvm/test/Transforms/Inline/optimization-remarks.ll @@ -19,10 +19,10 @@ ; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \ ; RUN: FileCheck -check-prefixes=CHECK,HOTNESS,ALWAYS %s -; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \ +; RUN: opt < %s -passes=inliner-wrapper-no-mandatory-first -pass-remarks=inline -pass-remarks-missed=inline \ ; RUN: -pass-remarks-analysis=inline -S 2>&1 | \ ; RUN: FileCheck -check-prefixes=CHECK,NO_HOTNESS,ALWAYS %s -; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \ +; RUN: opt < %s -passes=inliner-wrapper-no-mandatory-first -pass-remarks=inline -pass-remarks-missed=inline \ ; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \ ; RUN: FileCheck -check-prefixes=CHECK,HOTNESS,ALWAYS %s diff --git a/llvm/test/Transforms/Inline/pr46945.ll b/llvm/test/Transforms/Inline/pr46945.ll --- a/llvm/test/Transforms/Inline/pr46945.ll +++ b/llvm/test/Transforms/Inline/pr46945.ll @@ -1,12 +1,6 @@ -; RUN: opt %s -o - -S -passes=always-inliner-wrapper | FileCheck %s ; RUN: opt %s -o - -S -passes='default' | FileCheck %s -; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s -check-prefix=BASELINE +; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s -; In the baseline case, a will be first inlined into b, which makes c recursive, -; and, thus, un-inlinable. We need a baseline case to make sure intra-SCC order -; is as expected: b first, then a. - -; BASELINE: call void @b() ; CHECK-NOT: call void @b() define void @b() alwaysinline { entry: