Index: llvm/trunk/include/llvm/InitializePasses.h =================================================================== --- llvm/trunk/include/llvm/InitializePasses.h +++ llvm/trunk/include/llvm/InitializePasses.h @@ -66,7 +66,7 @@ void initializeAddressSanitizerPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); -void initializeAlwaysInlinerPass(PassRegistry&); +void initializeAlwaysInlinerLegacyPassPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); void initializeAssumptionCacheTrackerPass(PassRegistry &); void initializeAtomicExpandPass(PassRegistry&); Index: llvm/trunk/include/llvm/LinkAllPasses.h =================================================================== --- llvm/trunk/include/llvm/LinkAllPasses.h +++ llvm/trunk/include/llvm/LinkAllPasses.h @@ -39,6 +39,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/ObjCARC.h" @@ -97,7 +98,7 @@ (void) llvm::createInstrProfilingLegacyPass(); (void) llvm::createFunctionImportPass(); (void) llvm::createFunctionInliningPass(); - (void) llvm::createAlwaysInlinerPass(); + (void) llvm::createAlwaysInlinerLegacyPass(); (void) llvm::createGlobalDCEPass(); (void) llvm::createGlobalOptimizerPass(); (void) llvm::createGlobalsAAWrapperPass(); Index: llvm/trunk/include/llvm/Transforms/IPO.h =================================================================== --- llvm/trunk/include/llvm/Transforms/IPO.h +++ llvm/trunk/include/llvm/Transforms/IPO.h @@ -107,12 +107,6 @@ Pass *createFunctionInliningPass(InlineParams &Params); //===----------------------------------------------------------------------===// -/// createAlwaysInlinerPass - Return a new pass object that inlines only -/// functions that are marked as "always_inline". -Pass *createAlwaysInlinerPass(); -Pass *createAlwaysInlinerPass(bool InsertLifetime); - -//===----------------------------------------------------------------------===// /// createPruneEHPass - Return a new pass object which transforms invoke /// instructions into calls, if the callee can _not_ unwind the stack. /// Index: llvm/trunk/include/llvm/Transforms/IPO/AlwaysInliner.h =================================================================== --- llvm/trunk/include/llvm/Transforms/IPO/AlwaysInliner.h +++ llvm/trunk/include/llvm/Transforms/IPO/AlwaysInliner.h @@ -0,0 +1,40 @@ +//===-- AlwaysInliner.h - Pass to inline "always_inline" functions --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Provides passes to inlining "always_inline" functions. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H +#define LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Inlines functions marked as "always_inline". +/// +/// Note that this does not inline call sites marked as always_inline and does +/// not delete the functions even when all users are inlined. The normal +/// inliner should be used to handle call site inlining, this pass's goal is to +/// be the simplest possible pass to remove always_inline function definitions' +/// uses by inlining them. The \c GlobalDCE pass can be used to remove these +/// functions once all users are gone. +struct AlwaysInlinerPass : PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &); +}; + +/// Create a legacy pass manager instance of a pass to inline and remove +/// functions marked as "always_inline". +Pass *createAlwaysInlinerLegacyPass(bool InsertLifetime = true); + +} + +#endif // LLVM_TRANSFORMS_IPO_ALWAYSINLINER_H Index: llvm/trunk/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/trunk/lib/Passes/PassBuilder.cpp +++ llvm/trunk/lib/Passes/PassBuilder.cpp @@ -59,6 +59,7 @@ #include "llvm/Support/Regex.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/GCOVProfiler.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/ConstantMerge.h" #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" Index: llvm/trunk/lib/Passes/PassRegistry.def =================================================================== --- llvm/trunk/lib/Passes/PassRegistry.def +++ llvm/trunk/lib/Passes/PassRegistry.def @@ -38,6 +38,7 @@ #ifndef MODULE_PASS #define MODULE_PASS(NAME, CREATE_PASS) #endif +MODULE_PASS("always-inline", AlwaysInlinerPass()) MODULE_PASS("constmerge", ConstantMergePass()) MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass()) MODULE_PASS("deadargelim", DeadArgumentEliminationPass()) Index: llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -29,6 +29,7 @@ #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar/GVN.h" #include "llvm/Transforms/Vectorize.h" @@ -360,7 +361,7 @@ // Function calls are not supported, so make sure we inline everything. addPass(createAMDGPUAlwaysInlinePass()); - addPass(createAlwaysInlinerPass()); + addPass(createAlwaysInlinerLegacyPass()); // We need to add the barrier noop pass, otherwise adding the function // inlining pass will cause all of the PassConfigs passes to be run // one function at a time, which means if we have a nodule with two Index: llvm/trunk/lib/Transforms/IPO/AlwaysInliner.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/AlwaysInliner.cpp +++ llvm/trunk/lib/Transforms/IPO/AlwaysInliner.cpp @@ -0,0 +1,127 @@ +//===- InlineAlways.cpp - Code to inline always_inline functions ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements a custom inliner that handles only functions that +// are marked as "always inline". +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Analysis/InlineCost.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/CallingConv.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/Transforms/IPO/InlinerPass.h" +#include "llvm/Transforms/Utils/Cloning.h" + +using namespace llvm; + +#define DEBUG_TYPE "inline" + +PreservedAnalyses AlwaysInlinerPass::run(Module &M, ModuleAnalysisManager &) { + InlineFunctionInfo IFI; + SmallSetVector Calls; + bool Changed = false; + for (Function &F : M) + if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) && + isInlineViable(F)) { + Calls.clear(); + + for (User *U : F.users()) + if (auto CS = CallSite(U)) + if (CS.getCalledFunction() == &F) + Calls.insert(CS); + + for (CallSite CS : Calls) + // FIXME: We really shouldn't be able to fail to inline at this point! + // We should do something to log or check the inline failures here. + Changed |= InlineFunction(CS, IFI); + } + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} + +namespace { + +/// Inliner pass which only handles "always inline" functions. +/// +/// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner +/// base class to provide several facilities such as array alloca merging. +class AlwaysInlinerLegacyPass : public Inliner { + +public: + AlwaysInlinerLegacyPass() : Inliner(ID, /*InsertLifetime*/ true) { + initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); + } + + AlwaysInlinerLegacyPass(bool InsertLifetime) : Inliner(ID, InsertLifetime) { + initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); + } + + /// Main run interface method. We override here to avoid calling skipSCC(). + bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); } + + static char ID; // Pass identification, replacement for typeid + + InlineCost getInlineCost(CallSite CS) override; + + using llvm::Pass::doFinalization; + bool doFinalization(CallGraph &CG) override { + return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true); + } +}; +} + +char AlwaysInlinerLegacyPass::ID = 0; +INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", + "Inliner for always_inline functions", false, false) +INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) +INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass) +INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", + "Inliner for always_inline functions", false, false) + +Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { + return new AlwaysInlinerLegacyPass(InsertLifetime); +} + +/// \brief Get the inline cost for the always-inliner. +/// +/// The always inliner *only* handles functions which are marked with the +/// attribute to force inlining. As such, it is dramatically simpler and avoids +/// using the powerful (but expensive) inline cost analysis. Instead it uses +/// a very simple and boring direct walk of the instructions looking for +/// impossible-to-inline constructs. +/// +/// Note, it would be possible to go to some lengths to cache the information +/// computed here, but as we only expect to do this for relatively few and +/// small functions which have the explicit attribute to force inlining, it is +/// likely not worth it in practice. +InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallSite CS) { + Function *Callee = CS.getCalledFunction(); + + // Only inline direct calls to functions with always-inline attributes + // that are viable for inlining. FIXME: We shouldn't even get here for + // declarations. + if (Callee && !Callee->isDeclaration() && + CS.hasFnAttr(Attribute::AlwaysInline) && isInlineViable(*Callee)) + return InlineCost::getAlways(); + + return InlineCost::getNever(); +} Index: llvm/trunk/lib/Transforms/IPO/CMakeLists.txt =================================================================== --- llvm/trunk/lib/Transforms/IPO/CMakeLists.txt +++ llvm/trunk/lib/Transforms/IPO/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_library(LLVMipo + AlwaysInliner.cpp ArgumentPromotion.cpp BarrierNoopPass.cpp ConstantMerge.cpp @@ -14,7 +15,6 @@ IPConstantPropagation.cpp IPO.cpp InferFunctionAttrs.cpp - InlineAlways.cpp InlineSimple.cpp Inliner.cpp Internalize.cpp Index: llvm/trunk/lib/Transforms/IPO/IPO.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/IPO.cpp +++ llvm/trunk/lib/Transforms/IPO/IPO.cpp @@ -18,6 +18,7 @@ #include "llvm/InitializePasses.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" using namespace llvm; @@ -32,7 +33,7 @@ initializeGlobalDCELegacyPassPass(Registry); initializeGlobalOptLegacyPassPass(Registry); initializeIPCPPass(Registry); - initializeAlwaysInlinerPass(Registry); + initializeAlwaysInlinerLegacyPassPass(Registry); initializeSimpleInlinerPass(Registry); initializeInferFunctionAttrsLegacyPassPass(Registry); initializeInternalizeLegacyPassPass(Registry); @@ -82,7 +83,7 @@ } void LLVMAddAlwaysInlinerPass(LLVMPassManagerRef PM) { - unwrap(PM)->add(llvm::createAlwaysInlinerPass()); + unwrap(PM)->add(llvm::createAlwaysInlinerLegacyPass()); } void LLVMAddGlobalDCEPass(LLVMPassManagerRef PM) { Index: llvm/trunk/lib/Transforms/IPO/InlineAlways.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/InlineAlways.cpp +++ llvm/trunk/lib/Transforms/IPO/InlineAlways.cpp @@ -1,102 +0,0 @@ -//===- InlineAlways.cpp - Code to inline always_inline functions ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements a custom inliner that handles only functions that -// are marked as "always inline". -// -//===----------------------------------------------------------------------===// - -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/Analysis/AssumptionCache.h" -#include "llvm/Analysis/CallGraph.h" -#include "llvm/Analysis/InlineCost.h" -#include "llvm/Analysis/ProfileSummaryInfo.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/IR/CallSite.h" -#include "llvm/IR/CallingConv.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicInst.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Type.h" -#include "llvm/Transforms/IPO.h" -#include "llvm/Transforms/IPO/InlinerPass.h" - -using namespace llvm; - -#define DEBUG_TYPE "inline" - -namespace { - -/// \brief Inliner pass which only handles "always inline" functions. -class AlwaysInliner : public Inliner { - -public: - AlwaysInliner() : Inliner(ID, /*InsertLifetime*/ true) { - initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry()); - } - - AlwaysInliner(bool InsertLifetime) : Inliner(ID, InsertLifetime) { - initializeAlwaysInlinerPass(*PassRegistry::getPassRegistry()); - } - - /// Main run interface method. We override here to avoid calling skipSCC(). - bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); } - - static char ID; // Pass identification, replacement for typeid - - InlineCost getInlineCost(CallSite CS) override; - - using llvm::Pass::doFinalization; - bool doFinalization(CallGraph &CG) override { - return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true); - } -}; -} - -char AlwaysInliner::ID = 0; -INITIALIZE_PASS_BEGIN(AlwaysInliner, "always-inline", - "Inliner for always_inline functions", false, false) -INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) -INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass) -INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) -INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) -INITIALIZE_PASS_END(AlwaysInliner, "always-inline", - "Inliner for always_inline functions", false, false) - -Pass *llvm::createAlwaysInlinerPass() { return new AlwaysInliner(); } - -Pass *llvm::createAlwaysInlinerPass(bool InsertLifetime) { - return new AlwaysInliner(InsertLifetime); -} - -/// \brief Get the inline cost for the always-inliner. -/// -/// The always inliner *only* handles functions which are marked with the -/// attribute to force inlining. As such, it is dramatically simpler and avoids -/// using the powerful (but expensive) inline cost analysis. Instead it uses -/// a very simple and boring direct walk of the instructions looking for -/// impossible-to-inline constructs. -/// -/// Note, it would be possible to go to some lengths to cache the information -/// computed here, but as we only expect to do this for relatively few and -/// small functions which have the explicit attribute to force inlining, it is -/// likely not worth it in practice. -InlineCost AlwaysInliner::getInlineCost(CallSite CS) { - Function *Callee = CS.getCalledFunction(); - - // Only inline direct calls to functions with always-inline attributes - // that are viable for inlining. FIXME: We shouldn't even get here for - // declarations. - if (Callee && !Callee->isDeclaration() && - CS.hasFnAttr(Attribute::AlwaysInline) && isInlineViable(*Callee)) - return InlineCost::getAlways(); - - return InlineCost::getNever(); -} Index: llvm/trunk/lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- llvm/trunk/lib/Transforms/Utils/InlineFunction.cpp +++ llvm/trunk/lib/Transforms/Utils/InlineFunction.cpp @@ -1053,7 +1053,7 @@ /// If the inlined function has non-byval align arguments, then /// add @llvm.assume-based alignment assumptions to preserve this information. static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) { - if (!PreserveAlignmentAssumptions) + if (!PreserveAlignmentAssumptions || !IFI.GetAssumptionCache) return; AssumptionCache *AC = IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*CS.getCaller()) Index: llvm/trunk/test/Transforms/Inline/always-inline.ll =================================================================== --- llvm/trunk/test/Transforms/Inline/always-inline.ll +++ llvm/trunk/test/Transforms/Inline/always-inline.ll @@ -1,8 +1,12 @@ -; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s +; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL ; ; Ensure the threshold has no impact on these decisions. -; RUN: opt < %s -inline-threshold=20000000 -always-inline -S | FileCheck %s -; RUN: opt < %s -inline-threshold=-20000000 -always-inline -S | FileCheck %s +; RUN: opt < %s -inline-threshold=20000000 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL +; RUN: opt < %s -inline-threshold=-20000000 -always-inline -S | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-CALL +; +; The new pass manager doesn't re-use any threshold based infrastructure for +; the always inliner, but test that we get the correct result. +; RUN: opt < %s -passes=always-inline -S | FileCheck %s --check-prefix=CHECK define i32 @inner1() alwaysinline { ret i32 1 @@ -126,10 +130,23 @@ ret i32 1 } define i32 @outer7() { -; CHECK-LABEL: @outer7( -; CHECK-NOT: call -; CHECK: ret +; CHECK-CALL-LABEL: @outer7( +; CHECK-CALL-NOT: call +; CHECK-CALL: ret %r = call i32 @inner7() alwaysinline ret i32 %r } + +define float* @inner8(float* nocapture align 128 %a) alwaysinline { + ret float* %a +} +define float @outer8(float* nocapture %a) { +; CHECK-LABEL: @outer8( +; CHECK-NOT: call float* @inner8 +; CHECK: ret + + %inner_a = call float* @inner8(float* %a) + %f = load float, float* %inner_a, align 4 + ret float %f +} Index: llvm/trunk/tools/bugpoint/bugpoint.cpp =================================================================== --- llvm/trunk/tools/bugpoint/bugpoint.cpp +++ llvm/trunk/tools/bugpoint/bugpoint.cpp @@ -27,6 +27,7 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/Support/Valgrind.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" //Enable this macro to debug bugpoint itself. @@ -179,7 +180,7 @@ if (OptLevelO1 || OptLevelO2 || OptLevelO3) { PassManagerBuilder Builder; if (OptLevelO1) - Builder.Inliner = createAlwaysInlinerPass(); + Builder.Inliner = createAlwaysInlinerLegacyPass(); else if (OptLevelOs || OptLevelO2) Builder.Inliner = createFunctionInliningPass(2, OptLevelOs ? 1 : 0); else Index: llvm/trunk/tools/opt/opt.cpp =================================================================== --- llvm/trunk/tools/opt/opt.cpp +++ llvm/trunk/tools/opt/opt.cpp @@ -51,6 +51,7 @@ #include "llvm/Support/ToolOutputFile.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Coroutines.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/Cloning.h" #include @@ -259,7 +260,7 @@ } else if (OptLevel > 1) { Builder.Inliner = createFunctionInliningPass(OptLevel, SizeLevel); } else { - Builder.Inliner = createAlwaysInlinerPass(); + Builder.Inliner = createAlwaysInlinerLegacyPass(); } Builder.DisableUnitAtATime = !UnitAtATime; Builder.DisableUnrollLoops = (DisableLoopUnrolling.getNumOccurrences() > 0) ?