Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -66,7 +66,7 @@ void initializeAddressSanitizerPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); -void initializeAlwaysInlinerPass(PassRegistry&); +void initializeAlwaysInlinerLegacyPassPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); void initializeAssumptionCacheTrackerPass(PassRegistry &); void initializeAtomicExpandPass(PassRegistry&); Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -39,6 +39,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/ObjCARC.h" @@ -97,7 +98,7 @@ (void) llvm::createInstrProfilingLegacyPass(); (void) llvm::createFunctionImportPass(); (void) llvm::createFunctionInliningPass(); - (void) llvm::createAlwaysInlinerPass(); + (void) llvm::createAlwaysInlinerLegacyPass(); (void) llvm::createGlobalDCEPass(); (void) llvm::createGlobalOptimizerPass(); (void) llvm::createGlobalsAAWrapperPass(); Index: include/llvm/Transforms/IPO.h =================================================================== --- include/llvm/Transforms/IPO.h +++ include/llvm/Transforms/IPO.h @@ -105,12 +105,6 @@ Pass *createFunctionInliningPass(unsigned OptLevel, unsigned SizeOptLevel); //===----------------------------------------------------------------------===// -/// createAlwaysInlinerPass - Return a new pass object that inlines only -/// functions that are marked as "always_inline". -Pass *createAlwaysInlinerPass(); -Pass *createAlwaysInlinerPass(bool InsertLifetime); - -//===----------------------------------------------------------------------===// /// createPruneEHPass - Return a new pass object which transforms invoke /// instructions into calls, if the callee can _not_ unwind the stack. /// Index: include/llvm/Transforms/IPO/AlwaysInliner.h =================================================================== --- /dev/null +++ include/llvm/Transforms/IPO/AlwaysInliner.h @@ -0,0 +1,39 @@ +//===-- AlwaysInliner.h - Pass to inline "always_inline" functions --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Provides passes to inlining "always_inline" functions. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H +#define LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Inlines and removes functions marked as "always_inline". +/// +/// These functions are required to be inlined and deleted before generating +/// code. We do this even at O0 and so this pass works to do so using as simple +/// and straight forward of mechanisms as possible. +/// +/// However, because of the requirement to *remove* these functions, this pass is somewhat +struct AlwaysInlinerPass : PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &); +}; + +/// Create a legacy pass manager instance of a pass to inline and remove +/// functions marked as "always_inline". +Pass *createAlwaysInlinerLegacyPass(bool InsertLifetime = true); + +} + +#endif // LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -58,6 +58,7 @@ #include "llvm/Support/Regex.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/GCOVProfiler.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/ConstantMerge.h" #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" Index: lib/Passes/PassRegistry.def =================================================================== --- lib/Passes/PassRegistry.def +++ lib/Passes/PassRegistry.def @@ -37,6 +37,7 @@ #ifndef MODULE_PASS #define MODULE_PASS(NAME, CREATE_PASS) #endif +MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("constmerge", ConstantMergePass()) MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass()) MODULE_PASS("deadargelim", DeadArgumentEliminationPass()) Index: lib/Target/AMDGPU/AMDGPUTargetMachine.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -37,6 +37,7 @@ #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_os_ostream.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Vectorize.h" @@ -365,7 +366,7 @@ // Function calls are not supported, so make sure we inline everything. addPass(createAMDGPUAlwaysInlinePass()); - addPass(createAlwaysInlinerPass()); + addPass(createAlwaysInlinerLegacyPass()); // We need to add the barrier noop pass, otherwise adding the function // inlining pass will cause all of the PassConfigs passes to be run // one function at a time, which means if we have a nodule with two Index: lib/Transforms/IPO/AlwaysInliner.cpp =================================================================== --- lib/Transforms/IPO/AlwaysInliner.cpp +++ lib/Transforms/IPO/AlwaysInliner.cpp @@ -12,7 +12,8 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/ADT/SetVector.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/InlineCost.h" @@ -25,25 +26,51 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" -#include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/InlinerPass.h" +#include "llvm/Transforms/Utils/Cloning.h" using namespace llvm; #define DEBUG_TYPE "inline" +PreservedAnalyses AlwaysInlinerPass::run(Module &M, ModuleAnalysisManager &) { + InlineFunctionInfo IFI; + SmallSetVector Calls; + bool Changed = false; + for (Function &F : M) + if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) && + isInlineViable(F)) { + Calls.clear(); + + for (User *U : F.users()) + if (auto CS = CallSite(U)) + if (CS.getCalledFunction() == &F) + Calls.insert(CS); + + for (CallSite CS : Calls) + // FIXME: We really shouldn't be able to fail to inline at this point! + // We should do something to log or check the inline failures here. + Changed |= InlineFunction(CS, IFI); + } + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + namespace { -/// \brief Inliner pass which only handles "always inline" functions. -class AlwaysInliner : public Inliner { +/// Inliner pass which only handles "always inline" functions. +/// +/// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner +/// base class to provide several facilities such as array alloca merging. +class AlwaysInlinerLegacyPass : public Inliner { public: - AlwaysInliner() : Inliner(ID, /*InsertLifetime*/ true) { - initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry()); + AlwaysInlinerLegacyPass() : Inliner(ID, /*InsertLifetime*/ true) { + initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); } - AlwaysInliner(bool InsertLifetime) : Inliner(ID, InsertLifetime) { - initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry()); + AlwaysInlinerLegacyPass(bool InsertLifetime) : Inliner(ID, InsertLifetime) { + initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); } /// Main run interface method. We override here to avoid calling skipSCC(). @@ -60,20 +87,18 @@ }; } -char AlwaysInliner::ID = 0; -INITIALIZE_PASS_BEGIN(AlwaysInliner, "always-inline", +char AlwaysInlinerLegacyPass::ID = 0; +INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", "Inliner for always_inline functions", false, false) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) -INITIALIZE_PASS_END(AlwaysInliner, "always-inline", +INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", "Inliner for always_inline functions", false, false) -Pass *llvm::createAlwaysInlinerPass() { return new AlwaysInliner(); } - -Pass *llvm::createAlwaysInlinerPass(bool InsertLifetime) { - return new AlwaysInliner(InsertLifetime); +Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { + return new AlwaysInlinerLegacyPass(InsertLifetime); } /// \brief Get the inline cost for the always-inliner. @@ -88,7 +113,7 @@ /// computed here, but as we only expect to do this for relatively few and /// small functions which have the explicit attribute to force inlining, it is /// likely not worth it in practice. -InlineCost AlwaysInliner::getInlineCost(CallSite CS) { +InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallSite CS) { Function *Callee = CS.getCalledFunction(); // Only inline direct calls to functions with always-inline attributes Index: lib/Transforms/IPO/CMakeLists.txt =================================================================== --- lib/Transforms/IPO/CMakeLists.txt +++ lib/Transforms/IPO/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_library(LLVMipo + AlwaysInliner.cpp ArgumentPromotion.cpp BarrierNoopPass.cpp ConstantMerge.cpp @@ -14,7 +15,6 @@ IPConstantPropagation.cpp IPO.cpp InferFunctionAttrs.cpp - InlineAlways.cpp InlineSimple.cpp Inliner.cpp Internalize.cpp Index: lib/Transforms/IPO/IPO.cpp =================================================================== --- lib/Transforms/IPO/IPO.cpp +++ lib/Transforms/IPO/IPO.cpp @@ -18,6 +18,7 @@ #include "llvm/InitializePasses.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" using namespace llvm; @@ -32,7 +33,7 @@ initializeGlobalDCELegacyPassPass(Registry); initializeGlobalOptLegacyPassPass(Registry); initializeIPCPPass(Registry); - initializeAlwaysInlinerPass(Registry); + initializeAlwaysInlinerLegacyPassPass(Registry); initializeSimpleInlinerPass(Registry); initializeInferFunctionAttrsLegacyPassPass(Registry); initializeInternalizeLegacyPassPass(Registry); @@ -82,7 +83,7 @@ } void LLVMAddAlwaysInlinerPass(LLVMPassManagerRef PM) { - unwrap(PM)->add(llvm::createAlwaysInlinerPass()); + unwrap(PM)->add(llvm::createAlwaysInlinerLegacyPass()); } void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM) { Index: test/Transforms/Inline/always-inline.ll =================================================================== --- test/Transforms/Inline/always-inline.ll +++ test/Transforms/Inline/always-inline.ll @@ -1,8 +1,12 @@ -; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s +; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL ; ; Ensure the threshold has no impact on these decisions. -; RUN: opt < %s -inline-threshold=20000000 -always-inline -S | FileCheck %s -; RUN: opt < %s -inline-threshold=-20000000 -always-inline -S | FileCheck %s +; RUN: opt < %s -inline-threshold=20000000 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL +; RUN: opt < %s -inline-threshold=-20000000 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL +; +; The new pass manager doesn't re-use any threshold based infrastructure for +; the always inliner, but test that we get the correct result. +; RUN: opt < %s -passes=always-inline -S | FileCheck %s --check-prefix=CHECK define i32 @inner1() alwaysinline { ret i32 1 @@ -126,9 +130,9 @@ ret i32 1 } define i32 @outer7() { -; CHECK-LABEL: @outer7( -; CHECK-NOT: call -; CHECK: ret +; CHECK-CALL-LABEL: @outer7( +; CHECK-CALL-NOT: call +; CHECK-CALL: ret %r = call i32 @inner7() alwaysinline ret i32 %r Index: tools/bugpoint/bugpoint.cpp =================================================================== --- tools/bugpoint/bugpoint.cpp +++ tools/bugpoint/bugpoint.cpp @@ -27,6 +27,7 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Valgrind.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" //Enable this macro to debug bugpoint itself. @@ -179,7 +180,7 @@ if (OptLevelO1 || OptLevelO2 || OptLevelO3) { PassManagerBuilder Builder; if (OptLevelO1) - Builder.Inliner = createAlwaysInlinerPass(); + Builder.Inliner = createAlwaysInlinerLegacyPass(); else if (OptLevelOs || OptLevelO2) Builder.Inliner = createFunctionInliningPass(2, OptLevelOs ? 1 : 0); else Index: tools/opt/opt.cpp =================================================================== --- tools/opt/opt.cpp +++ tools/opt/opt.cpp @@ -51,6 +51,7 @@ #include "llvm/Support/ToolOutputFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Coroutines.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/Cloning.h" #include @@ -259,7 +260,7 @@ } else if (OptLevel > 1) { Builder.Inliner = createFunctionInliningPass(OptLevel, SizeLevel); } else { - Builder.Inliner = createAlwaysInlinerPass(); + Builder.Inliner = createAlwaysInlinerLegacyPass(); } Builder.DisableUnitAtATime = !UnitAtATime; Builder.DisableUnrollLoops = (DisableLoopUnrolling.getNumOccurrences() > 0) ?