diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -195,6 +195,7 @@ void initializeInstNamerPass(PassRegistry&); void initializeInstSimplifyLegacyPassPass(PassRegistry &); void initializeInstrProfilingLegacyPassPass(PassRegistry&); +void initializeCGProfileLegacyPassPass(PassRegistry&); void initializeInstrOrderFileLegacyPassPass(PassRegistry&); void initializeInstructionCombiningPassPass(PassRegistry&); void initializeInstructionSelectPass(PassRegistry&); diff --git a/llvm/include/llvm/Transforms/IPO.h b/llvm/include/llvm/Transforms/IPO.h --- a/llvm/include/llvm/Transforms/IPO.h +++ b/llvm/include/llvm/Transforms/IPO.h @@ -282,6 +282,7 @@ ModulePass *createWriteThinLTOBitcodePass(raw_ostream &Str, raw_ostream *ThinLinkOS = nullptr); +ModulePass *createCGProfileLegacyPass(); } // End llvm namespace #endif diff --git a/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h b/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h --- a/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h +++ b/llvm/include/llvm/Transforms/Instrumentation/CGProfile.h @@ -19,11 +19,6 @@ class CGProfilePass : public PassInfoMixin { public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); - -private: - void addModuleFlags( - Module &M, - MapVector, uint64_t> &Counts) const; }; } // end namespace llvm diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp --- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp +++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp @@ -834,6 +834,9 @@ if (MergeFunctions) MPM.add(createMergeFunctionsPass()); + // Add Module flag "CG Profile" based on Branch Frequency Information. + MPM.add(createCGProfileLegacyPass()); + // LoopSink pass sinks instructions hoisted by LICM, which serves as a // canonicalization pass that enables other optimizations. As a result, // LoopSink pass needs to be a very late IR pass to avoid undoing LICM diff --git a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp --- a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp +++ b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp @@ -15,17 +15,43 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/Transforms/IPO.h" #include "llvm/Transforms/Instrumentation.h" #include using namespace llvm; -PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) { +namespace { +static bool +addModuleFlags(Module &M, + MapVector, uint64_t> &Counts) { + if (Counts.empty()) + return false; + + LLVMContext &Context = M.getContext(); + MDBuilder MDB(Context); + std::vector Nodes; + + for (auto E : Counts) { + Metadata *Vals[] = {ValueAsMetadata::get(E.first.first), + ValueAsMetadata::get(E.first.second), + MDB.createConstant(ConstantInt::get( + Type::getInt64Ty(Context), E.second))}; + Nodes.push_back(MDNode::get(Context, Vals)); + } + + M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes)); + return true; +} + +static bool +runCGProfilePass(Module &M, + function_ref GetBFI, + function_ref GetTTI) { MapVector, uint64_t> Counts; - FunctionAnalysisManager &FAM = - MAM.getResult(M).getManager(); InstrProfSymtab Symtab; auto UpdateCounts = [&](TargetTransformInfo &TTI, Function *F, Function *CalledF, uint64_t NewCount) { @@ -35,14 +61,14 @@ Count = SaturatingAdd(Count, NewCount); }; // Ignore error here. Indirect calls are ignored if this fails. - (void)(bool)Symtab.create(M); + (void)(bool) Symtab.create(M); for (auto &F : M) { if (F.isDeclaration()) continue; - auto &BFI = FAM.getResult(F); + auto &BFI = GetBFI(F); if (BFI.getEntryFreq() == 0) continue; - TargetTransformInfo &TTI = FAM.getResult(F); + TargetTransformInfo &TTI = GetTTI(F); for (auto &BB : F) { Optional BBCount = BFI.getBlockProfileCount(&BB); if (!BBCount) @@ -69,28 +95,55 @@ } } - addModuleFlags(M, Counts); - - return PreservedAnalyses::all(); + return addModuleFlags(M, Counts); } -void CGProfilePass::addModuleFlags( - Module &M, - MapVector, uint64_t> &Counts) const { - if (Counts.empty()) - return; +struct CGProfileLegacyPass : public ModulePass { + static char ID; + CGProfileLegacyPass() : ModulePass(ID) { + initializeCGProfileLegacyPassPass(*PassRegistry::getPassRegistry()); + } - LLVMContext &Context = M.getContext(); - MDBuilder MDB(Context); - std::vector Nodes; + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addRequired(); + } - for (auto E : Counts) { - Metadata *Vals[] = {ValueAsMetadata::get(E.first.first), - ValueAsMetadata::get(E.first.second), - MDB.createConstant(ConstantInt::get( - Type::getInt64Ty(Context), E.second))}; - Nodes.push_back(MDNode::get(Context, Vals)); + bool runOnModule(Module &M) override { + auto GetBFI = [this](Function &F) -> BlockFrequencyInfo & { + return this->getAnalysis(F).getBFI(); + }; + auto GetTTI = [this](Function &F) -> TargetTransformInfo & { + return this->getAnalysis().getTTI(F); + }; + + return runCGProfilePass(M, GetBFI, GetTTI); } +}; - M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes)); +} // namespace + +char CGProfileLegacyPass::ID = 0; + +INITIALIZE_PASS(CGProfileLegacyPass, "cgprofile", "Call Graph Profile", false, + false) + +ModulePass *llvm::createCGProfileLegacyPass() { + return new CGProfileLegacyPass(); +} + +PreservedAnalyses CGProfilePass::run(Module &M, ModuleAnalysisManager &MAM) { + FunctionAnalysisManager &FAM = + MAM.getResult(M).getManager(); + auto GetBFI = [&FAM](Function &F) -> BlockFrequencyInfo & { + return FAM.getResult(F); + }; + auto GetTTI = [&FAM](Function &F) -> TargetTransformInfo & { + return FAM.getResult(F); + }; + + runCGProfilePass(M, GetBFI, GetTTI); + + return PreservedAnalyses::all(); } diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp --- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp @@ -112,6 +112,7 @@ initializePGOInstrumentationUseLegacyPassPass(Registry); initializePGOIndirectCallPromotionLegacyPassPass(Registry); initializePGOMemOPSizeOptLegacyPassPass(Registry); + initializeCGProfileLegacyPassPass(Registry); initializeInstrOrderFileLegacyPassPass(Registry); initializeInstrProfilingLegacyPassPass(Registry); initializeMemorySanitizerLegacyPassPass(Registry); diff --git a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll --- a/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/opt-pipeline.ll @@ -276,6 +276,13 @@ ; GCN-O1-NEXT: Warn about non-applied transformations ; GCN-O1-NEXT: Alignment from assumptions ; GCN-O1-NEXT: Strip Unused Function Prototypes +; GCN-O1-NEXT: Call Graph Profile +; GCN-O1-NEXT: FunctionPass Manager +; GCN-O1-NEXT: Dominator Tree Construction +; GCN-O1-NEXT: Natural Loop Information +; GCN-O1-NEXT: Post-Dominator Tree Construction +; GCN-O1-NEXT: Branch Probability Analysis +; GCN-O1-NEXT: Block Frequency Analysis ; GCN-O1-NEXT: FunctionPass Manager ; GCN-O1-NEXT: Dominator Tree Construction ; GCN-O1-NEXT: Natural Loop Information @@ -623,6 +630,13 @@ ; GCN-O2-NEXT: Strip Unused Function Prototypes ; GCN-O2-NEXT: Dead Global Elimination ; GCN-O2-NEXT: Merge Duplicate Global Constants +; GCN-O2-NEXT: Call Graph Profile +; GCN-O2-NEXT: FunctionPass Manager +; GCN-O2-NEXT: Dominator Tree Construction +; GCN-O2-NEXT: Natural Loop Information +; GCN-O2-NEXT: Post-Dominator Tree Construction +; GCN-O2-NEXT: Branch Probability Analysis +; GCN-O2-NEXT: Block Frequency Analysis ; GCN-O2-NEXT: FunctionPass Manager ; GCN-O2-NEXT: Dominator Tree Construction ; GCN-O2-NEXT: Natural Loop Information @@ -975,6 +989,13 @@ ; GCN-O3-NEXT: Strip Unused Function Prototypes ; GCN-O3-NEXT: Dead Global Elimination ; GCN-O3-NEXT: Merge Duplicate Global Constants +; GCN-O3-NEXT: Call Graph Profile +; GCN-O3-NEXT: FunctionPass Manager +; GCN-O3-NEXT: Dominator Tree Construction +; GCN-O3-NEXT: Natural Loop Information +; GCN-O3-NEXT: Post-Dominator Tree Construction +; GCN-O3-NEXT: Branch Probability Analysis +; GCN-O3-NEXT: Block Frequency Analysis ; GCN-O3-NEXT: FunctionPass Manager ; GCN-O3-NEXT: Dominator Tree Construction ; GCN-O3-NEXT: Natural Loop Information diff --git a/llvm/test/Instrumentation/cgprofile.ll b/llvm/test/Instrumentation/cgprofile.ll --- a/llvm/test/Instrumentation/cgprofile.ll +++ b/llvm/test/Instrumentation/cgprofile.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -passes cg-profile -S | FileCheck %s +; RUN: opt < %s -cgprofile -S | FileCheck %s declare void @b() diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll --- a/llvm/test/Other/opt-O2-pipeline.ll +++ b/llvm/test/Other/opt-O2-pipeline.ll @@ -280,6 +280,13 @@ ; CHECK-NEXT: Strip Unused Function Prototypes ; CHECK-NEXT: Dead Global Elimination ; CHECK-NEXT: Merge Duplicate Global Constants +; CHECK-NEXT: Call Graph Profile +; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Dominator Tree Construction +; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Post-Dominator Tree Construction +; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Natural Loop Information diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll --- a/llvm/test/Other/opt-O3-pipeline.ll +++ b/llvm/test/Other/opt-O3-pipeline.ll @@ -285,6 +285,13 @@ ; CHECK-NEXT: Strip Unused Function Prototypes ; CHECK-NEXT: Dead Global Elimination ; CHECK-NEXT: Merge Duplicate Global Constants +; CHECK-NEXT: Call Graph Profile +; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Dominator Tree Construction +; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Post-Dominator Tree Construction +; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Natural Loop Information diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll --- a/llvm/test/Other/opt-Os-pipeline.ll +++ b/llvm/test/Other/opt-Os-pipeline.ll @@ -266,6 +266,13 @@ ; CHECK-NEXT: Strip Unused Function Prototypes ; CHECK-NEXT: Dead Global Elimination ; CHECK-NEXT: Merge Duplicate Global Constants +; CHECK-NEXT: Call Graph Profile +; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Dominator Tree Construction +; CHECK-NEXT: Natural Loop Information +; CHECK-NEXT: Post-Dominator Tree Construction +; CHECK-NEXT: Branch Probability Analysis +; CHECK-NEXT: Block Frequency Analysis ; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Natural Loop Information