[Inliner] Attempt to inline calls to alwaysinline functions first

[aeubanks]

Summary

For PR46945 we added an extra alwaysinline inliner pass that runs before
the normal inliner pass. This was to fix the issue where we have two
mutually recursive functions, one marked alwaysinline, but due to inline
ordering, the other function may have been inlined into the alwaysinline
function first, causing the alwaysinline function to not be inlined into
the other function.

Adding a whole new pass is quite a bit of work. Rather, we can just
choose to handle alwaysinline calls first before inlining normal calls.

[aeubanks]

I'll update the tests if this lg

clang/test/Frontend/optimization-remark-line-directive.c

 // This file tests -Rpass diagnostics together with #line
 // directives. We cannot map #line directives back to
 // a SourceLocation.
 
 // RUN: %clang_cc1 %s -Rpass=inline -debug-info-kind=line-tables-only -emit-llvm-only -verify -fno-experimental-new-pass-manager
 
 // The new PM inliner is not added to the default pipeline at O0, so we add
 // some optimizations to trigger it.
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -mllvm -mandatory-inlining-first=false -verify
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify
 
 int foo(int x, int y) __attribute__((always_inline));
 int foo(int x, int y) { return x + y; }
 
 // expected-remark@+2 {{'foo' inlined into 'bar'}} expected-note@+2 {{could not determine the original source location for /bad/path/to/original.c:1230:25}}
 #line 1230 "/bad/path/to/original.c"
 int bar(int j) { return foo(j, j - 2); }

clang/test/Frontend/optimization-remark-new-pm.c

 // Verify that remarks for the inliner appear. The remarks under the new PM will
 // be slightly different than those emitted by the legacy PM. The new PM inliner
 // also doesnot appear to be added at O0, so we test at O1.
-// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -mllvm -mandatory-inlining-first=false -verify
-// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -mllvm -mandatory-inlining-first=false -verify
+// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify
+// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify
 
 int foo(int x, int y) __attribute__((always_inline));
 int foo(int x, int y) { return x + y; }
 
 float foz(int x, int y) __attribute__((noinline));
 float foz(int x, int y) { return x * y; }
 
 // The negative diagnostics are emitted twice because the inliner runs
 // twice.
 //
 int bar(int j) {
   // expected-remark@+2 {{'foz' not inlined into 'bar' because it should never be inlined (cost=never)}}
   // expected-remark@+1 {{'foo' inlined into 'bar'}}
   return foo(j, j - 2) * foz(j - 2, j);
 }

clang/test/Frontend/optimization-remark-with-hotness-new-pm.c

 // RUN:     %S/Inputs/optimization-remark-with-hotness-sample.proftext \
 // RUN:     -o %t-sample.profdata
 //
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
-// RUN:     -fdiagnostics-show-hotness -verify -mllvm -mandatory-inlining-first=false
+// RUN:     -fdiagnostics-show-hotness -verify
 // The clang version of the previous test.
 // RUN: %clang -target x86_64-apple-macosx10.9 %s -c -emit-llvm -o /dev/null \
 // RUN:     -fprofile-instr-use=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
-// RUN:     -fdiagnostics-show-hotness -Xclang -verify -mllvm -mandatory-inlining-first=false
+// RUN:     -fdiagnostics-show-hotness -Xclang -verify
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-sample-use=%t-sample.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
 // RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 \
-// RUN:     -verify -mllvm -mandatory-inlining-first=false
+// RUN:     -verify
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
-// RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify -mllvm -mandatory-inlining-first=false
+// RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
-// RUN:     -Rpass-analysis=inline -mllvm -mandatory-inlining-first=false 2>&1 | FileCheck -check-prefix=HOTNESS_OFF %s
+// RUN:     -Rpass-analysis=inline 2>&1 | FileCheck -check-prefix=HOTNESS_OFF %s
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
-// RUN:     -Rpass-analysis=inline -Rno-pass-with-hotness -mllvm -mandatory-inlining-first=false 2>&1 | FileCheck \
+// RUN:     -Rpass-analysis=inline -Rno-pass-with-hotness 2>&1 | FileCheck \
 // RUN:     -check-prefix=HOTNESS_OFF %s
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -Rpass-analysis=inline -fdiagnostics-show-hotness \
-// RUN:     -fdiagnostics-hotness-threshold=100 -mllvm -mandatory-inlining-first=false 2>&1 \
+// RUN:     -fdiagnostics-hotness-threshold=100 2>&1 \
 // RUN:     | FileCheck -allow-empty -check-prefix=THRESHOLD %s
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -Rpass=inline -Rpass-analysis=inline \
-// RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -mllvm -mandatory-inlining-first=false 2>&1 \
+// RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 2>&1 \
 // RUN:     | FileCheck -check-prefix=NO_PGO %s
 
 int foo(int x, int y) __attribute__((always_inline));
 int foo(int x, int y) { return x + y; }
 
 int sum = 0;
 
 void bar(int x) {

clang/test/Frontend/optimization-remark.c

 // Check that we can override -Rpass= with -Rno-pass.
 // RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 // RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS
 // RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS
 // RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // The inliner for the new PM does not seem to be enabled at O0, but we still
 // get the same remarks with at least O1.
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -mllvm -mandatory-inlining-first=false -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -mllvm -mandatory-inlining-first=false -o -  2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o -  2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // Check that -w doesn't disable remarks.
 // RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -mllvm -mandatory-inlining-first=false -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // -Reverything implies -Rpass=.*.
 // RUN: %clang_cc1 %s -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // -Rpass implies -Rpass=.*
 // RUN: %clang_cc1 %s -Rpass -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 
 // CHECK-REMARKS: remark:

llvm/include/llvm/Analysis/InlineAdvisor.h

 /// Interface for deciding whether to inline a call site or not.
 class InlineAdvisor {
 public:
   InlineAdvisor(InlineAdvisor &&) = delete;
   virtual ~InlineAdvisor();
 
   /// Get an InlineAdvice containing a recommendation on whether to
   /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to
-  /// be up-to-date wrt previous inlining decisions. \p MandatoryOnly indicates
-  /// only mandatory (always-inline) call sites should be recommended - this
-  /// allows the InlineAdvisor track such inlininings.
+  /// be up-to-date wrt previous inlining decisions.
   /// Returns an InlineAdvice with the inlining recommendation.
-  std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB,
-                                          bool MandatoryOnly = false);
+  std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB);
 
   /// This must be called when the Inliner pass is entered, to allow the
   /// InlineAdvisor update internal state, as result of function passes run
   /// between Inliner pass runs (for the same module).
   virtual void onPassEntry() {}
 
   /// This must be called when the Inliner pass is exited, as function passes
   /// may be run subsequently. This allows an implementation of InlineAdvisor
   /// to prepare for a partial update.
   virtual void onPassExit() {}
 
 protected:
   InlineAdvisor(Module &M, FunctionAnalysisManager &FAM);
   virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0;
-  virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB,
-                                                           bool Advice);
 
   Module &M;
   FunctionAnalysisManager &FAM;
   std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats;
 
   /// We may want to defer deleting functions to after the inlining for a whole
   /// module has finished. This allows us to reliably use function pointers as
   /// unique identifiers, as an efficient implementation detail of the
   /// InlineAdvisor. Otherwise, it is possible the memory allocator
   /// re-allocate Function objects at the same address of a deleted Function;
   /// and Functions are potentially created during the function passes called
   /// after each SCC inlining (e.g. argument promotion does that).
   void freeDeletedFunctions();
 
   bool isFunctionDeleted(const Function *F) const {
     return DeletedFunctions.count(F);
   }
 
-  enum class MandatoryInliningKind { NotMandatory, Always, Never };
-
-  static MandatoryInliningKind getMandatoryKind(CallBase &CB,
-                                                FunctionAnalysisManager &FAM,
-                                                OptimizationRemarkEmitter &ORE);
-
   OptimizationRemarkEmitter &getCallerORE(CallBase &CB);
 
 private:
   friend class InlineAdvice;
   void markFunctionAsDeleted(Function *F);
   std::unordered_set<const Function *> DeletedFunctions;
 };
 

llvm/include/llvm/Analysis/InlineOrder.h

public:
   using const_reference = const T &;
 
   virtual ~InlineOrder() {}
 
   virtual size_t size() = 0;
 
   virtual void push(const T &Elt) = 0;
 
+  virtual void push_front(const T &Elt) = 0;
+
   virtual T pop() = 0;
 
   virtual const_reference front() = 0;
 
   virtual void erase_if(function_ref<bool(T)> Pred) = 0;
 
   bool empty() { return !size(); }
 };
 
 template <typename T, typename Container = SmallVector<T, 16>>
 class DefaultInlineOrder : public InlineOrder<T> {
   using reference = T &;
   using const_reference = const T &;
 
 public:
   size_t size() override { return Calls.size() - FirstIndex; }
 
   void push(const T &Elt) override { Calls.push_back(Elt); }
 
+  void push_front(const T &Elt) override {
+    Calls.insert(Calls.begin() + FirstIndex, Elt);
+  }
+
   T pop() override {
     assert(size() > 0);
     return Calls[FirstIndex++];
   }
 
   const_reference front() override {
     assert(size() > 0);
     return Calls[FirstIndex];
void push(const T &Elt) override {
     const int InlineHistoryID = Elt.second;
     const PriorityT Goodness = PriorityT::evaluate(CB);
 
     Heap.push_back({CB, Goodness});
     std::push_heap(Heap.begin(), Heap.end(), cmp);
     InlineHistoryMap[CB] = InlineHistoryID;
   }
 
+  void push_front(const T &Elt) override { push(Elt); }
+
   T pop() override {
     assert(size() > 0);
     adjust();
 
     CallBase *CB = Heap.front().first;
     T Result = std::make_pair(CB, InlineHistoryMap[CB]);
     InlineHistoryMap.erase(CB);
     std::pop_heap(Heap.begin(), Heap.end(), cmp);

llvm/include/llvm/Transforms/IPO/Inliner.h

 /// It should be noted that the legacy inliners do considerably more than this
 /// inliner pass does. They provide logic for manually merging allocas, and
 /// doing considerable DCE including the DCE of dead functions. This pass makes
 /// every attempt to be simpler. DCE of functions requires complex reasoning
 /// about comdat groups, etc. Instead, it is expected that other more focused
 /// passes be composed to achieve the same end result.
 class InlinerPass : public PassInfoMixin<InlinerPass> {
 public:
-  InlinerPass(bool OnlyMandatory = false) : OnlyMandatory(OnlyMandatory) {}
+  InlinerPass() = default;
   InlinerPass(InlinerPass &&Arg) = default;
 
   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
                         LazyCallGraph &CG, CGSCCUpdateResult &UR);
 
-  void printPipeline(raw_ostream &OS,
-                     function_ref<StringRef(StringRef)> MapClassName2PassName);
-
 private:
   InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
                             FunctionAnalysisManager &FAM, Module &M);
   std::unique_ptr<InlineAdvisor> OwnedAdvisor;
-  const bool OnlyMandatory;
 };
 
 /// Module pass, wrapping the inliner pass. This works in conjunction with the
 /// InlineAdvisorAnalysis to facilitate inlining decisions taking into account
 /// module-wide state, that need to keep track of inter-inliner pass runs, for
 /// a given module. An InlineAdvisor is configured and kept alive for the
 /// duration of the ModuleInlinerWrapperPass::run.
 class ModuleInlinerWrapperPass
     : public PassInfoMixin<ModuleInlinerWrapperPass> {
 public:
   ModuleInlinerWrapperPass(
-      InlineParams Params = getInlineParams(), bool MandatoryFirst = true,
+      InlineParams Params = getInlineParams(),
       InliningAdvisorMode Mode = InliningAdvisorMode::Default,
       unsigned MaxDevirtIterations = 0);
   ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default;
 
   PreservedAnalyses run(Module &, ModuleAnalysisManager &);
 
   /// Allow adding more CGSCC passes, besides inlining. This should be called
   /// before run is called, as part of pass pipeline building.

llvm/lib/Analysis/InlineAdvisor.cpp

if (ImportedFunctionsStats) {
     assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
     ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
                                  InlinerFunctionImportStatsOpts::Verbose);
   }
 
   freeDeletedFunctions();
 }
 
-std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
-                                                                bool Advice) {
-  return std::make_unique<InlineAdvice>(this, CB, getCallerORE(CB), Advice);
-}
-
-InlineAdvisor::MandatoryInliningKind
-InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
-                                OptimizationRemarkEmitter &ORE) {
-  auto &Callee = *CB.getCalledFunction();
-
-  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
-    return FAM.getResult<TargetLibraryAnalysis>(F);
-  };
-
-  auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
-
-  auto TrivialDecision =
-      llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
-
-  if (TrivialDecision.hasValue()) {
-    if (TrivialDecision->isSuccess())
-      return MandatoryInliningKind::Always;
-    else
-      return MandatoryInliningKind::Never;
-  }
-  return MandatoryInliningKind::NotMandatory;
-}
-
-std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
-                                                       bool MandatoryOnly) {
-  if (!MandatoryOnly)
-    return getAdviceImpl(CB);
-  bool Advice = CB.getCaller() != CB.getCalledFunction() &&
-                MandatoryInliningKind::Always ==
-                    getMandatoryKind(CB, FAM, getCallerORE(CB));
-  return getMandatoryAdvice(CB, Advice);
+std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB) {
 }
 
 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
   return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller());
 }

llvm/lib/Passes/PassBuilder.cpp

if (ParamName == OptionName) {
           formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName)
               .str(),
           inconvertibleErrorCode());
     }
   }
   return Result;
 }
 
-Expected<bool> parseInlinerPassOptions(StringRef Params) {
-  return parseSinglePassOption(Params, "only-mandatory", "InlinerPass");
-}
-
 Expected<bool> parseEarlyCSEPassOptions(StringRef Params) {
   return parseSinglePassOption(Params, "memssa", "EarlyCSE");
 }
 
 Expected<bool> parseEntryExitInstrumenterPassOptions(StringRef Params) {
   return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter");
 }
 

llvm/lib/Passes/PassBuilderPipelines.cpp

static cl::opt<bool>
     EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
                             cl::Hidden,
                             cl::desc("Enable inline deferral during PGO"));
 
 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
                                        cl::Hidden, cl::ZeroOrMore,
                                        cl::desc("Enable memory profiler"));
 
-static cl::opt<bool> PerformMandatoryInliningsFirst(
-    "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
-    cl::desc("Perform mandatory inlinings module-wide, before performing "
-             "inlining."));
-
 static cl::opt<bool> EnableO3NonTrivialUnswitching(
     "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
     cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"));
 
 PipelineTuningOptions::PipelineTuningOptions() {
   LoopInterleaving = true;
   LoopVectorization = true;
   SLPVectorization = false;
PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
   InlineParams IP = getInlineParamsFromOptLevel(Level);
   if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
       PGOOpt->Action == PGOOptions::SampleUse)
     IP.HotCallSiteThreshold = 0;
 
   if (PGOOpt)
     IP.EnableDeferral = EnablePGOInlineDeferral;
 
-  ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
-                                UseInlineAdvisor, MaxDevirtIterations);
+  ModuleInlinerWrapperPass MIWP(IP, UseInlineAdvisor, MaxDevirtIterations);
 
   // Require the GlobalsAA analysis for the module so we can query it within
   // the CGSCC pipeline.
   MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
   // Invalidate AAManager so it can be recreated and pick up the newly available
   // GlobalsAA.
   MIWP.addModulePass(
       createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));

llvm/lib/Passes/PassRegistry.def

 MODULE_PASS("function-import", FunctionImportPass())
 MODULE_PASS("function-specialization", FunctionSpecializationPass())
 MODULE_PASS("globaldce", GlobalDCEPass())
 MODULE_PASS("globalopt", GlobalOptPass())
 MODULE_PASS("globalsplit", GlobalSplitPass())
 MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
 MODULE_PASS("inferattrs", InferFunctionAttrsPass())
 MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
-MODULE_PASS("inliner-wrapper-no-mandatory-first", ModuleInlinerWrapperPass(
-  getInlineParams(),
-  false))
 MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfiling())
 MODULE_PASS("internalize", InternalizePass())
 MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 MODULE_PASS("ipsccp", IPSCCPPass())
 MODULE_PASS("iroutliner", IROutlinerPass())
 MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(dbgs()))

 #endif
 CGSCC_PASS("argpromotion", ArgumentPromotionPass())
 CGSCC_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 CGSCC_PASS("function-attrs", PostOrderFunctionAttrsPass())
 CGSCC_PASS("attributor-cgscc", AttributorCGSCCPass())
 CGSCC_PASS("openmp-opt-cgscc", OpenMPOptCGSCCPass())
 CGSCC_PASS("coro-split", CoroSplitPass())
 CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass())
+CGSCC_PASS("inline", InlinerPass())
 #undef CGSCC_PASS
 
 #ifndef CGSCC_PASS_WITH_PARAMS
 #define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
-CGSCC_PASS_WITH_PARAMS("inline",
-                       "InlinerPass",
-                       [](bool OnlyMandatory) {
-                         return InlinerPass(OnlyMandatory);
-                       },
-                       parseInlinerPassOptions,
-                       "only-mandatory")
 #undef CGSCC_PASS_WITH_PARAMS
 
 #ifndef FUNCTION_ANALYSIS
 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)
 #endif
 FUNCTION_ANALYSIS("aa", AAManager())
 FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis())
 FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis())

llvm/lib/Transforms/IPO/Inliner.cpp

for (auto &N : InitialC) {
     // We want to generally process call sites top-down in order for
     // simplifications stemming from replacing the call with the returned value
     // after inlining to be visible to subsequent inlining decisions.
     // FIXME: Using instructions sequence is a really bad way to do this.
     // Instead we should do an actual RPO walk of the function body.
     for (Instruction &I : instructions(N.getFunction()))
       if (auto *CB = dyn_cast<CallBase>(&I))
         if (Function *Callee = CB->getCalledFunction()) {
-          if (!Callee->isDeclaration())
+          if (!Callee->isDeclaration()) {
+            if (Callee->hasFnAttribute(Attribute::AlwaysInline))
+              Calls->push_front({CB, -1});
+            else
               Calls->push({CB, -1});
-          else if (!isa<IntrinsicInst>(I)) {
+          } else if (!isa<IntrinsicInst>(I)) {
             using namespace ore;
             setInlineRemark(*CB, "unavailable definition");
             ORE.emit([&]() {
               return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
                      << NV("Callee", Callee) << " will not be inlined into "
                      << NV("Caller", CB->getCaller())
                      << " because its definition is unavailable"
                      << setIsVerbose();
while (!Calls->empty() && Calls->front().first->getCaller() == &F) {
           UR.InlinedInternalEdges.count({&N, C})) {
         LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node "
                              "previously split out of this SCC by inlining: "
                           << F.getName() << " -> " << Callee.getName() << "\n");
         setInlineRemark(*CB, "recursive SCC split");
         continue;
       }
 
-      auto Advice = Advisor.getAdvice(*CB, OnlyMandatory);
+      auto Advice = Advisor.getAdvice(*CB);
       // Check whether we want to inline this callsite.
       if (!Advice->isInliningRecommended()) {
         Advice->recordUnattemptedInlining();
         continue;
       }
 
       // Setup the data structure used to plumb customization into the
       // `InlineFunction` routine.
while (!Calls->empty() && Calls->front().first->getCaller() == &F) {
             // Try to promote an indirect (virtual) call without waiting for
             // the post-inline cleanup and the next DevirtSCCRepeatedPass
             // iteration because the next iteration may not happen and we may
             // miss inlining it.
             if (tryPromoteCall(*ICB))
               NewCallee = ICB->getCalledFunction();
           }
           if (NewCallee)
-            if (!NewCallee->isDeclaration())
+            if (!NewCallee->isDeclaration()) {
+              if (NewCallee->hasFnAttribute(Attribute::AlwaysInline))
+                Calls->push_front({ICB, NewHistoryID});
+              else
                 Calls->push({ICB, NewHistoryID});
             }
         }
+      }
 
       // Merge the attributes based on the inlining.
       AttributeFuncs::mergeAttributesForInlining(F, Callee);
 
       // For local functions, check whether this makes the callee trivially
       // dead. In that case, we can drop the body of the function eagerly
       // which may reduce the number of callers of other functions to one,
       // changing inline cost thresholds.
PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
   // Even if we change the IR, we update the core CGSCC data structures and so
   // can preserve the proxy to the function analysis manager.
   PreservedAnalyses PA;
   PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
   return PA;
 }
 
 ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params,
-                                                   bool MandatoryFirst,
                                                    InliningAdvisorMode Mode,
                                                    unsigned MaxDevirtIterations)
     : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations),
       PM(), MPM() {
   // Run the inliner first. The theory is that we are walking bottom-up and so
   // the callees have already been fully optimized, and we want to inline them
   // into the callers so that our optimizations can reflect that.
   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
   // because it makes profile annotation in the backend inaccurate.
-  if (MandatoryFirst)
-    PM.addPass(InlinerPass(/*OnlyMandatory*/ true));
   PM.addPass(InlinerPass());
 }
 
 PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
                                                 ModuleAnalysisManager &MAM) {
   auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
   if (!IAA.tryCreate(Params, Mode, CGSCCInlineReplayFile)) {
     M.getContext().emitError(
PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
   MPM.run(M, MAM);
 
   IAA.clear();
 
   // The ModulePassManager has already taken care of invalidating analyses.
   return PreservedAnalyses::all();
 }
 
-void InlinerPass::printPipeline(
-    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
-  static_cast<PassInfoMixin<InlinerPass> *>(this)->printPipeline(
-      OS, MapClassName2PassName);
-  if (OnlyMandatory)
-    OS << "<only-mandatory>";
-}
-
 void ModuleInlinerWrapperPass::printPipeline(
     raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
   // Print some info about passes added to the wrapper. This is however
   // incomplete as InlineAdvisorAnalysis part isn't included (which also depends
   // on Params and Mode).
   if (!MPM.isEmpty()) {
     MPM.printPipeline(OS, MapClassName2PassName);
     OS << ",";

llvm/test/Other/new-pm-print-pipeline.ll

 ; CHECK-17: function(print<stack-lifetime><may>,print<stack-lifetime><must>)
 
 ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='function(simplifycfg<bonus-inst-threshold=5;forward-switch-cond;switch-to-lookup;keep-loops;hoist-common-insts;sink-common-insts>,simplifycfg<bonus-inst-threshold=7;no-forward-switch-cond;no-switch-to-lookup;no-keep-loops;no-hoist-common-insts;no-sink-common-insts>)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-18
 ; CHECK-18: function(simplifycfg<bonus-inst-threshold=5;forward-switch-cond;switch-to-lookup;keep-loops;hoist-common-insts;sink-common-insts>,simplifycfg<bonus-inst-threshold=7;no-forward-switch-cond;no-switch-to-lookup;no-keep-loops;no-hoist-common-insts;no-sink-common-insts>)
 
 ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='function(loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only>,loop-vectorize<interleave-forced-only;vectorize-forced-only>)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-19
 ; CHECK-19: function(loop-vectorize<no-interleave-forced-only;no-vectorize-forced-only;>,loop-vectorize<interleave-forced-only;vectorize-forced-only;>)
 
-; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='inliner-wrapper,inliner-wrapper-no-mandatory-first' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-20
-; CHECK-20: cgscc(inline<only-mandatory>,inline),cgscc(inline)
+; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='inliner-wrapper' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-20
+; CHECK-20: cgscc(inline)
 
 ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='scc-oz-module-inliner' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-21
 ; CHECK-21: require<globals-aa>,function(invalidate<aa>),require<profile-summary>,cgscc(devirt<4>(inline<only-mandatory>,inline,{{.*}},instcombine{{.*}}))

llvm/test/Transforms/Inline/inline_nossp.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes='cgscc(inline)' %s -S -pass-remarks-missed=inline 2>&1 | FileCheck --check-prefixes=CHECK,CHECK-INLINE %s
 ; RUN: opt -passes=always-inline -o - -S %s | FileCheck %s
 
-; CHECK-INLINE: 'ssp' not inlined into 'nossp_caller' because it should never be inlined (cost=never): stack protected callee but caller requested no stack protector
-; CHECK-INLINE: 'nossp' not inlined into 'ssp_caller' because it should never be inlined (cost=never): stack protected caller but callee requested no stack protector
+; CHECK-INLINE: 'ssp' not inlined into 'nossp_caller1' because it should never be inlined (cost=never): stack protected callee but caller requested no stack protector
+; CHECK-INLINE: 'nossp' not inlined into 'ssp_caller1' because it should never be inlined (cost=never): stack protected caller but callee requested no stack protector
 
-; Not interesting to test.
-define i32 @nossp() { ret i32 41 }
-define i32 @ssp() sspstrong { ret i32 42 }
-define i32 @nossp_alwaysinline() alwaysinline { ret i32 43 }
-define i32 @ssp_alwaysinline() sspstrong alwaysinline { ret i32 44 }
+define void @nossp() {
+; CHECK-LABEL: @nossp(
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+define void @ssp() sspstrong {
+; CHECK-LABEL: @ssp(
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+define void @nossp_alwaysinline() alwaysinline {
+; CHECK-LABEL: @nossp_alwaysinline(
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+define void @ssp_alwaysinline() sspstrong alwaysinline {
+; CHECK-LABEL: @ssp_alwaysinline(
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
 
 ; @ssp should not be inlined due to mismatch stack protector.
+define void @nossp_caller1() {
+; CHECK-LABEL: @nossp_caller1(
+; CHECK-NEXT:    call void @ssp()
+; CHECK-NEXT:    ret void
+;
+  call void @ssp()
+  ret void
+}
+
 ; @ssp_alwaysinline should be inlined due to alwaysinline.
-define i32 @nossp_caller() {
-; CHECK-LABEL: @nossp_caller(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @ssp()
-; CHECK-NEXT:    ret i32 44
-;
-  call i32 @ssp()
-  %2 = call i32 @ssp_alwaysinline()
-  ret i32 %2
+define void @nossp_caller2() {
+; CHECK-LABEL: @nossp_caller2(
+; CHECK-NEXT:    ret void
+;
+  call void @ssp_alwaysinline()
+  ret void
 }
+
 ; @nossp should not be inlined due to mismatch stack protector.
+define void @ssp_caller1() sspstrong {
+; CHECK-LABEL: @ssp_caller1(
+; CHECK-NEXT:    call void @nossp()
+; CHECK-NEXT:    ret void
+;
+  call void @nossp()
+  ret void
+}
+
 ; @nossp_alwaysinline should be inlined due to alwaysinline.
-define i32 @ssp_caller() sspstrong {
-; CHECK-LABEL: @ssp_caller(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @nossp()
-; CHECK-NEXT:    ret i32 43
-;
-  call i32 @nossp()
-  %2 = call i32 @nossp_alwaysinline()
-  ret i32 %2
+define void @ssp_caller2() sspstrong {
+; CHECK-LABEL: @ssp_caller2(
+; CHECK-NEXT:    ret void
+;
+  call void @nossp_alwaysinline()
+  ret void
 }
 
 ; The alwaysinline attribute can also appear on the CallBase (ie. the call
 ; site), ie. when __attribute__((flatten)) is used on the caller. Treat this
 ; the same as if the caller had the fn attr alwaysinline and permit inline
 ; substitution, despite the mismatch between caller and callee on ssp attrs.
 ;
 ; Curiously, the always_inline attribute on a CallInst is only expanded by the
 ; inline pass, but not always_inline pass!
-define i32 @nossp_alwaysinline_caller() {
+define void @nossp_alwaysinline_caller() {
 ; CHECK-INLINE-LABEL: @nossp_alwaysinline_caller(
-; CHECK-INLINE-NEXT:    ret i32 42
+; CHECK-INLINE-NEXT:    ret void
 ;
-  %1 = call i32 @ssp() alwaysinline
-  ret i32 %1
+  call void @ssp() alwaysinline
+  ret void
 }

llvm/test/Transforms/Inline/inline_stats.ll

 ; First with legacy PM
 ; RUN: opt -S -inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
 ; RUN: opt -S -inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
 
 ; Do again with new PM
 ; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
 ; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
 
-; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
-; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
-
 ; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
 ; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-VERBOSE,CHECK
 
 ; CHECK: ------- Dumping inliner stats for [<stdin>] -------
 ; CHECK-BASIC-NOT: -- List of inlined functions:
 ; CHECK-BASIC-NOT: -- Inlined not imported function
 ; CHECK-VERBOSE: -- List of inlined functions:
 ; CHECK-VERBOSE: Inlined not imported function [internal2]: #inlines = 6, #inlines_to_importing_module = 2

llvm/test/Transforms/Inline/no-inline-line-tables.ll

entry:
   %0 = load i32, i32* %x.addr, align 4, !dbg !14
   ret i32 %0, !dbg !14
 }
 
 ; Function Attrs: nounwind readnone speculatable willreturn
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 ; Function Attrs: alwaysinline nounwind
-define i32 @g(i32 %x) #0 !dbg !15 {
+define i32 @g(i32 %y) #0 !dbg !15 {
 entry:
-  %x.addr = alloca i32, align 4
-  store i32 %x, i32* %x.addr, align 4
-  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !16, metadata !DIExpression()), !dbg !17
+  %y.addr = alloca i32, align 4
+  store i32 %y, i32* %y.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %y.addr, metadata !16, metadata !DIExpression()), !dbg !17
   br label %L, !dbg !17
 
 L:                                                ; preds = %entry
   call void @llvm.dbg.label(metadata !18), !dbg !19
-  store i32 42, i32* %x.addr, align 4, !dbg !20
-  %0 = load i32, i32* %x.addr, align 4, !dbg !21
+  store i32 42, i32* %y.addr, align 4, !dbg !20
+  %0 = load i32, i32* %y.addr, align 4, !dbg !21
   ret i32 %0, !dbg !21
 }
 
 ; Function Attrs: nounwind readnone speculatable willreturn
 declare void @llvm.dbg.label(metadata) #1
 
 ; Check that debug info for inlined code uses the call location and that debug
 ; intrinsics are removed.
 ; Function Attrs: noinline nounwind optnone
 define i32 @main() #2 !dbg !22 {
 entry:
 ; CHECK-LABEL: @main()
 ; CHECK-NOT: @f
 ; CHECK-NOT: @llvm.dbg.declare
 ; CHECK: %{{[0-9]+}} = load i32, i32* %x.addr.i, align 4, !dbg ![[VAR1:[0-9]+]]
   %call = call i32 @f(i32 3), !dbg !25
 
 ; Another test for inlining debug intrinsics where the intrinsic appears at the
 ; start of the basic block.
 ; CHECK-NOT: @g
 ; CHECK-NOT: @llvm.dbg.label
-; CHECK: %{{[0-9]+}} = load i32, i32* %x.addr.i1, align 4, !dbg ![[VAR2:[0-9]+]]
+; CHECK: %{{[0-9]+}} = load i32, i32* %y.addr.i, align 4, !dbg ![[VAR2:[0-9]+]]
   %call1 = call i32 @g(i32 340), !dbg !26
   ret i32 0, !dbg !27
 }
 
 ; CHECK: ![[VAR1]] = !DILocation(line: 10, scope: ![[SCOPE:[0-9]+]])
 ; CHECK: ![[VAR2]] = !DILocation(line: 11, scope: ![[SCOPE]])
 
 attributes #0 = { alwaysinline nounwind "no-inline-line-tables" }

llvm/test/Transforms/Inline/optimization-remarks-with-hotness.ll

 ; RUN: opt < %s -inline -pass-remarks=inline -pass-remarks-missed=inline \
 ; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
 ; RUN:     | FileCheck %s
 ; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
 ; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
 ; RUN:     | FileCheck %s
-; RUN: opt < %s -passes=inliner-wrapper-no-mandatory-first -pass-remarks=inline -pass-remarks-missed=inline \
-; RUN:     -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
-; RUN:     | FileCheck %s
 
 ; CHECK: 'foo' inlined into 'bar' with (cost=always): always inline attribute (hotness: 30)
 ; CHECK: 'foz' not inlined into 'bar' because it should never be inlined (cost=never): noinline function attribute (hotness: 30)
 
 ; Function Attrs: alwaysinline nounwind uwtable
 define i32 @foo() #0 !prof !1 {
 entry:
   ret i32 4

llvm/test/Transforms/Inline/optimization-remarks.ll

 
 ; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
 ; RUN:       -pass-remarks-analysis=inline -S 2>&1 | \
 ; RUN:       FileCheck -check-prefixes=CHECK,NO_HOTNESS,ALWAYS %s
 ; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
 ; RUN:       -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
 ; RUN:       FileCheck -check-prefixes=CHECK,HOTNESS,ALWAYS %s
 
-; RUN: opt < %s -passes=inliner-wrapper-no-mandatory-first -pass-remarks=inline -pass-remarks-missed=inline \
-; RUN:       -pass-remarks-analysis=inline -S 2>&1 | \
-; RUN:       FileCheck -check-prefixes=CHECK,NO_HOTNESS,ALWAYS %s
-; RUN: opt < %s -passes=inliner-wrapper-no-mandatory-first -pass-remarks=inline -pass-remarks-missed=inline \
-; RUN:       -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
-; RUN:       FileCheck -check-prefixes=CHECK,HOTNESS,ALWAYS %s
-
 ; HOTNESS-DAG: fox will not be inlined into bar because its definition is unavailable
 ; NO_HOTNESS-NOT: fox will not be inlined into bar because its definition is unavailable
 ; ALWAYS-DAG: 'foo' inlined into 'bar' with (cost=always): always inline attribute
 ; CHECK-DAG: 'foz' not inlined into 'bar' because it should never be inlined (cost=never): noinline function attribute
 
 ; Function Attrs: alwaysinline nounwind uwtable
 define i32 @foo(i32 %x, i32 %y) #0 !prof !1 {
 entry: