Index: include/llvm/CodeGen/TargetLoweringObjectFileImpl.h =================================================================== --- include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -36,11 +36,14 @@ protected: MCSymbolRefExpr::VariantKind PLTRelativeVariantKind = MCSymbolRefExpr::VK_None; + const TargetMachine *TM; public: TargetLoweringObjectFileELF() = default; ~TargetLoweringObjectFileELF() override = default; + void Initialize(MCContext &Ctx, const TargetMachine &TM) override; + /// Emit Obj-C garbage collection and linker options. void emitModuleMetadata(MCStreamer &Streamer, Module &M) const override; Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -94,6 +94,7 @@ void initializeCFIInstrInserterPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); +void initializeCGProfilePassPass(PassRegistry&); void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphPrinterLegacyPassPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -80,6 +80,7 @@ (void) llvm::createCallGraphDOTPrinterPass(); (void) llvm::createCallGraphViewerPass(); (void) llvm::createCFGSimplificationPass(); + (void) llvm::createCGProfilePass(); (void) llvm::createCFLAndersAAWrapperPass(); (void) llvm::createCFLSteensAAWrapperPass(); (void) llvm::createStructurizeCFGPass(); Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -187,6 +187,8 @@ ModulePass *createSanitizerCoverageModulePass( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions()); +ModulePass *createCGProfilePass(); + /// Calculate what to divide by to scale counts. /// /// Given the maximum count, calculate a divisor that will scale all the Index: lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -91,6 +91,12 @@ // ELF //===----------------------------------------------------------------------===// +void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx, + const TargetMachine &TgtM) { + TargetLoweringObjectFile::Initialize(Ctx, TgtM); + TM = &TgtM; +} + void TargetLoweringObjectFileELF::emitModuleMetadata(MCStreamer &Streamer, Module &M) const { auto &C = getContext(); @@ -116,15 +122,49 @@ StringRef Section; GetObjCImageInfo(M, Version, Flags, Section); - if (Section.empty()) + if (!Section.empty()) { + auto *S = C.getELFSection(Section, ELF::SHT_PROGBITS, ELF::SHF_ALLOC); + Streamer.SwitchSection(S); + Streamer.EmitLabel(C.getOrCreateSymbol(StringRef("OBJC_IMAGE_INFO"))); + Streamer.EmitIntValue(Version, 4); + Streamer.EmitIntValue(Flags, 4); + Streamer.AddBlankLine(); + } + + SmallVector ModuleFlags; + M.getModuleFlagsMetadata(ModuleFlags); + + MDNode *CFGProfile = nullptr; + + for (const auto &MFE : ModuleFlags) { + StringRef Key = MFE.Key->getString(); + if (Key == "CG Profile") { + CFGProfile = cast(MFE.Val); + break; + } + } + + if (!CFGProfile) return; - auto *S = C.getELFSection(Section, ELF::SHT_PROGBITS, ELF::SHF_ALLOC); - Streamer.SwitchSection(S); - Streamer.EmitLabel(C.getOrCreateSymbol(StringRef("OBJC_IMAGE_INFO"))); - Streamer.EmitIntValue(Version, 4); - Streamer.EmitIntValue(Flags, 4); - Streamer.AddBlankLine(); + auto GetSym = [this](const MDOperand &MDO) { + auto V = cast(MDO); + const Function *F = cast(V->getValue()); + return TM->getSymbol(F); + }; + + for (const auto &Edge : CFGProfile->operands()) { + MDNode *E = cast(Edge); + const MCSymbol *From = GetSym(E->getOperand(0)); + const MCSymbol *To = GetSym(E->getOperand(1)); + uint64_t Count = cast(E->getOperand(2)) + ->getValue() + ->getUniqueInteger() + .getZExtValue(); + Streamer.emitCGProfileEntry( + MCSymbolRefExpr::create(From, MCSymbolRefExpr::VK_None, C), + MCSymbolRefExpr::create(To, MCSymbolRefExpr::VK_None, C), Count); + } } MCSymbol *TargetLoweringObjectFileELF::getCFIPersonalitySymbol( Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -409,6 +409,7 @@ void visitModuleFlag(const MDNode *Op, DenseMap &SeenIDs, SmallVectorImpl &Requirements); + void visitModuleFlagCGProfileEntry(const MDOperand &MDO); void visitFunction(const Function &F); void visitBasicBlock(BasicBlock &BB); void visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty); @@ -1411,6 +1412,25 @@ Assert(M.getNamedMetadata("llvm.linker.options"), "'Linker Options' named metadata no longer supported"); } + + if (ID->getString() == "CG Profile") { + for (const MDOperand &MDO : cast(Op->getOperand(2))->operands()) + visitModuleFlagCGProfileEntry(MDO); + } +} + +void Verifier::visitModuleFlagCGProfileEntry(const MDOperand &MDO) { + auto Node = dyn_cast_or_null(MDO); + Assert(Node && Node->getNumOperands() == 3, "expected a MDNode triple", MDO); + auto From = dyn_cast_or_null(Node->getOperand(0)); + Assert(From && isa(From->getValue()), "expected a Function", + Node->getOperand(0)); + auto To = dyn_cast_or_null(Node->getOperand(1)); + Assert(To && isa(To->getValue()), "expected a Function", + Node->getOperand(1)); + auto Count = dyn_cast_or_null(Node->getOperand(2)); + Assert(Count && Count->getType()->isIntegerTy(), + "expected an integer constant", Node->getOperand(2)); } /// Return true if this attribute kind only applies to functions. Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -694,6 +694,8 @@ MPM.add(createConstantMergePass()); // Merge dup global constants } + MPM.add(createCGProfilePass()); + if (MergeFunctions) MPM.add(createMergeFunctionsPass()); Index: lib/Transforms/Instrumentation/CGProfile.cpp =================================================================== --- /dev/null +++ lib/Transforms/Instrumentation/CGProfile.cpp @@ -0,0 +1,110 @@ +//===-- CGProfile.cpp -----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/MapVector.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Instrumentation.h" + +#include + +using namespace llvm; + +class CGProfilePass : public ModulePass { +public: + static char ID; + + CGProfilePass() : ModulePass(ID) { + initializeCGProfilePassPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { return "CGProfilePass"; } + +private: + bool runOnModule(Module &M) override; + bool addModuleFlags( + Module &M, + MapVector, uint64_t> &Counts) const; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + } +}; + +bool CGProfilePass::runOnModule(Module &M) { + if (skipModule(M)) + return false; + + MapVector, uint64_t> Counts; + + for (auto &F : M) { + if (F.isDeclaration()) + continue; + getAnalysis(F).getBPI(); + auto &BFI = getAnalysis(F).getBFI(); + for (const auto &BB : F) { + Optional BBCount = BFI.getBlockProfileCount(&BB); + if (!BBCount) + continue; + for (const auto &I : BB) { + auto *CI = dyn_cast(&I); + if (!CI) + continue; + Function *CalledF = CI->getCalledFunction(); + if (!CalledF || CalledF->isIntrinsic()) + continue; + + uint64_t &Count = Counts[std::make_pair(&F, CalledF)]; + Count = SaturatingAdd(Count, *BBCount); + } + } + } + + return addModuleFlags(M, Counts); +} + +bool CGProfilePass::addModuleFlags( + Module &M, + MapVector, uint64_t> &Counts) const { + if (Counts.empty()) + return false; + + LLVMContext &Context = M.getContext(); + MDBuilder MDB(Context); + std::vector Nodes; + + for (auto E : Counts) { + SmallVector Vals; + Vals.push_back(ValueAsMetadata::get(E.first.first)); + Vals.push_back(ValueAsMetadata::get(E.first.second)); + Vals.push_back(MDB.createConstant( + ConstantInt::get(Type::getInt64Ty(Context), E.second))); + Nodes.push_back(MDNode::get(Context, Vals)); + } + + M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes)); + return true; +} + +char CGProfilePass::ID = 0; +INITIALIZE_PASS_BEGIN(CGProfilePass, "cg-profile", + "Generate profile information from the call graph.", + false, false) +INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) +INITIALIZE_PASS_END(CGProfilePass, "cg-profile", + "Generate profile information from the call graph.", false, + false) + +ModulePass *llvm::createCGProfilePass() { return new CGProfilePass(); } Index: lib/Transforms/Instrumentation/CMakeLists.txt =================================================================== --- lib/Transforms/Instrumentation/CMakeLists.txt +++ lib/Transforms/Instrumentation/CMakeLists.txt @@ -1,6 +1,7 @@ add_llvm_library(LLVMInstrumentation AddressSanitizer.cpp BoundsChecking.cpp + CGProfile.cpp DataFlowSanitizer.cpp GCOVProfiling.cpp MemorySanitizer.cpp Index: lib/Transforms/Instrumentation/Instrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/Instrumentation.cpp +++ lib/Transforms/Instrumentation/Instrumentation.cpp @@ -60,6 +60,7 @@ initializeAddressSanitizerModulePass(Registry); initializeBoundsCheckingLegacyPassPass(Registry); initializeGCOVProfilerLegacyPassPass(Registry); + initializeCGProfilePassPass(Registry); initializePGOInstrumentationGenLegacyPassPass(Registry); initializePGOInstrumentationUseLegacyPassPass(Registry); initializePGOIndirectCallPromotionLegacyPassPass(Registry); Index: test/Instrumentation/cgprofile.ll =================================================================== --- /dev/null +++ test/Instrumentation/cgprofile.ll @@ -0,0 +1,28 @@ +; RUN: opt < %s -cg-profile -S | FileCheck %s + +declare void @b() + +define void @a() !prof !1 { + call void @b() + ret void +} + +define void @freq(i1 %cond) !prof !1 { + br i1 %cond, label %A, label %B, !prof !2 +A: + call void @a(); + ret void +B: + call void @b(); + ret void +} + +!1 = !{!"function_entry_count", i64 32} +!2 = !{!"branch_weights", i32 5, i32 10} + +; CHECK: !llvm.module.flags = !{![[cgprof:[0-9]+]]} +; CHECK: ![[cgprof]] = !{i32 5, !"CG Profile", ![[prof:[0-9]+]]} +; CHECK: ![[prof]] = !{![[e0:[0-9]+]], ![[e1:[0-9]+]], ![[e2:[0-9]+]]} +; CHECK: ![[e0]] = !{void ()* @a, void ()* @b, i64 32} +; CHECK: ![[e1]] = !{void (i1)* @freq, void ()* @a, i64 11} +; CHECK: ![[e2]] = !{void (i1)* @freq, void ()* @b, i64 20} Index: test/MC/ELF/cgprofile.ll =================================================================== --- /dev/null +++ test/MC/ELF/cgprofile.ll @@ -0,0 +1,50 @@ +; RUN: llc -filetype=asm %s -o - -mtriple x86_64-pc-linux-gnu | FileCheck %s +; RUN: llc -filetype=obj %s -o %t -mtriple x86_64-pc-linux-gnu +; RUN: llvm-readobj -elf-cg-profile %t | FileCheck %s --check-prefix=OBJ + +declare void @b() + +define void @a() { + call void @b() + ret void +} + +define void @freq(i1 %cond) { + br i1 %cond, label %A, label %B +A: + call void @a(); + ret void +B: + call void @b(); + ret void +} + +!llvm.module.flags = !{!0} + +!0 = !{i32 5, !"CG Profile", !1} +!1 = !{!2, !3, !4} +!2 = !{void ()* @a, void ()* @b, i64 32} +!3 = !{void (i1)* @freq, void ()* @a, i64 11} +!4 = !{void (i1)* @freq, void ()* @b, i64 20} + +; CHECK: .cg_profile a, b, 32 +; CHECK: .cg_profile freq, a, 11 +; CHECK: .cg_profile freq, b, 20 + +; OBJ: CGProfile [ +; OBJ: CGProfileEntry { +; OBJ: From: a +; OBJ: To: b +; OBJ: Weight: 32 +; OBJ: } +; OBJ: CGProfileEntry { +; OBJ: From: freq +; OBJ: To: a +; OBJ: Weight: 11 +; OBJ: } +; OBJ: CGProfileEntry { +; OBJ: From: freq +; OBJ: To: b +; OBJ: Weight: 20 +; OBJ: } +; OBJ:] Index: test/Verifier/module-flags-cgprofile.ll =================================================================== --- /dev/null +++ test/Verifier/module-flags-cgprofile.ll @@ -0,0 +1,30 @@ +; RUN: not llvm-as < %s -o /dev/null 2>&1 | FileCheck %s + +declare void @b() +declare void @a() + +!llvm.module.flags = !{!0} + +!0 = !{i32 5, !"CG Profile", !1} +!1 = !{!2, !"", !3, !4, !5, !6, !7, !8} +!2 = !{void ()* @a, void ()* @b, i64 32} +!3 = !{void ()* @a, void ()* @b} +!4 = !{void ()* @a, void ()* @b, i64 32, i64 32} +!5 = !{!"a", void ()* @b, i64 32} +!6 = !{void ()* @a, !"b", i64 32} +!7 = !{void ()* @a, void ()* @b, !""} +!8 = !{void ()* @a, void ()* @b, null} + +; CHECK: expected a MDNode triple +; CHECK: !"" +; CHECK: expected a MDNode triple +; CHECK: !3 = !{void ()* @a, void ()* @b} +; CHECK: expected a MDNode triple +; CHECK: !4 = !{void ()* @a, void ()* @b, i64 32, i64 32} +; CHECK: expected a Function +; CHECK: !"a" +; CHECK: expected a Function +; CHECK: !"b" +; CHECK: expected an integer constant +; CHECK: !"" +; CHECK: expected an integer constant