Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -88,6 +88,7 @@ void initializeCFGViewerLegacyPassPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); +void initializeCGProfilePassPass(PassRegistry&); void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphPrinterLegacyPassPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); Index: include/llvm/LinkAllPasses.h =================================================================== --- include/llvm/LinkAllPasses.h +++ include/llvm/LinkAllPasses.h @@ -76,6 +76,7 @@ (void) llvm::createCallGraphDOTPrinterPass(); (void) llvm::createCallGraphViewerPass(); (void) llvm::createCFGSimplificationPass(); + (void) llvm::createCGProfilePass(); (void) llvm::createCFLAndersAAWrapperPass(); (void) llvm::createCFLSteensAAWrapperPass(); (void) llvm::createStructurizeCFGPass(); Index: include/llvm/MC/MCAssembler.h =================================================================== --- include/llvm/MC/MCAssembler.h +++ include/llvm/MC/MCAssembler.h @@ -407,6 +407,13 @@ const MCLOHContainer &getLOHContainer() const { return const_cast(this)->getLOHContainer(); } + + struct CGProfileEntry { + const MCSymbol *From; + const MCSymbol *To; + uint64_t Count; + }; + std::vector CGProfile; /// @} /// \name Backend Data Access /// @{ Index: include/llvm/MC/MCELFStreamer.h =================================================================== --- include/llvm/MC/MCELFStreamer.h +++ include/llvm/MC/MCELFStreamer.h @@ -66,6 +66,9 @@ void EmitValueToAlignment(unsigned, int64_t, unsigned, unsigned) override; + void emitCGProfileEntry(const MCSymbol *From, const MCSymbol *To, + uint64_t Count) override; + void FinishImpl() override; void EmitBundleAlignMode(unsigned AlignPow2) override; Index: include/llvm/MC/MCStreamer.h =================================================================== --- include/llvm/MC/MCStreamer.h +++ include/llvm/MC/MCStreamer.h @@ -870,6 +870,9 @@ SMLoc Loc = SMLoc()); virtual void EmitWinEHHandlerData(SMLoc Loc = SMLoc()); + virtual void emitCGProfileEntry(const MCSymbol *From, const MCSymbol *To, + uint64_t Count); + /// Get the .pdata section used for the given section. Typically the given /// section is either the main .text section or some other COMDAT .text /// section, but it may be any section containing code. Index: include/llvm/Object/ELFTypes.h =================================================================== --- include/llvm/Object/ELFTypes.h +++ include/llvm/Object/ELFTypes.h @@ -40,6 +40,7 @@ template struct Elf_Hash_Impl; template struct Elf_GnuHash_Impl; template struct Elf_Chdr_Impl; +template struct Elf_CGProfile_Impl; template struct ELFType { private: @@ -66,6 +67,7 @@ using Hash = Elf_Hash_Impl>; using GnuHash = Elf_GnuHash_Impl>; using Chdr = Elf_Chdr_Impl>; + using CGProfile = Elf_CGProfile_Impl>; using DynRange = ArrayRef; using ShdrRange = ArrayRef; using SymRange = ArrayRef; @@ -551,6 +553,14 @@ Elf_Xword ch_addralign; }; +template +struct Elf_CGProfile_Impl { + LLVM_ELF_IMPORT_TYPES_ELFT(ELFT) + Elf_Word cgp_from; + Elf_Word cgp_to; + Elf_Xword cgp_weight; +}; + // MIPS .reginfo section template struct Elf_Mips_RegInfo; Index: include/llvm/Transforms/Instrumentation.h =================================================================== --- include/llvm/Transforms/Instrumentation.h +++ include/llvm/Transforms/Instrumentation.h @@ -186,6 +186,8 @@ ModulePass *createSanitizerCoverageModulePass( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions()); +ModulePass *createCGProfilePass(); + /// \brief Calculate what to divide by to scale counts. /// /// Given the maximum count, calculate a divisor that will scale all the Index: lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -98,16 +98,44 @@ StringRef Section; GetObjCImageInfo(M, Version, Flags, Section); - if (Section.empty()) + if (!Section.empty()) { + auto &C = getContext(); + auto *S = C.getELFSection(Section, ELF::SHT_PROGBITS, ELF::SHF_ALLOC); + Streamer.SwitchSection(S); + Streamer.EmitLabel(C.getOrCreateSymbol(StringRef("OBJC_IMAGE_INFO"))); + Streamer.EmitIntValue(Version, 4); + Streamer.EmitIntValue(Flags, 4); + Streamer.AddBlankLine(); + } + + SmallVector ModuleFlags; + M.getModuleFlagsMetadata(ModuleFlags); + + MDNode *CFGProfile = nullptr; + + for (const auto &MFE : ModuleFlags) { + StringRef Key = MFE.Key->getString(); + if (Key == "CFG Profile") { + CFGProfile = cast(MFE.Val); + break; + } + } + + if (!CFGProfile) return; - auto &C = getContext(); - auto *S = C.getELFSection(Section, ELF::SHT_PROGBITS, ELF::SHF_ALLOC); - Streamer.SwitchSection(S); - Streamer.EmitLabel(C.getOrCreateSymbol(StringRef("OBJC_IMAGE_INFO"))); - Streamer.EmitIntValue(Version, 4); - Streamer.EmitIntValue(Flags, 4); - Streamer.AddBlankLine(); + for (const auto &Edge : CFGProfile->operands()) { + MDNode *E = cast(Edge); + const MCSymbol *From = Streamer.getContext().getOrCreateSymbol( + cast(E->getOperand(0))->getString()); + const MCSymbol *To = Streamer.getContext().getOrCreateSymbol( + cast(E->getOperand(1))->getString()); + uint64_t Count = cast(E->getOperand(2)) + ->getValue() + ->getUniqueInteger() + .getZExtValue(); + Streamer.emitCGProfileEntry(From, To, Count); + } } MCSymbol *TargetLoweringObjectFileELF::getCFIPersonalitySymbol( Index: lib/MC/ELFObjectWriter.cpp =================================================================== --- lib/MC/ELFObjectWriter.cpp +++ lib/MC/ELFObjectWriter.cpp @@ -1299,6 +1299,13 @@ } } + MCSectionELF *CGProfileSection = nullptr; + if (!Asm.CGProfile.empty()) { + CGProfileSection = + Ctx.getELFSection(".note.llvm.cgprofile", ELF::SHT_NOTE, 0, 16, ""); + SectionIndexMap[CGProfileSection] = addToSectionTable(CGProfileSection); + } + for (MCSectionELF *Group : Groups) { align(Group->getAlignment()); @@ -1333,6 +1340,17 @@ SectionOffsets[RelSection] = std::make_pair(SecStart, SecEnd); } + if (CGProfileSection) { + uint64_t SecStart = getStream().tell(); + for (const MCAssembler::CGProfileEntry &CGPE : Asm.CGProfile) { + write32(CGPE.From->getIndex()); + write32(CGPE.To->getIndex()); + write64(CGPE.Count); + } + uint64_t SecEnd = getStream().tell(); + SectionOffsets[CGProfileSection] = std::make_pair(SecStart, SecEnd); + } + { uint64_t SecStart = getStream().tell(); const MCSectionELF *Sec = createStringTable(Ctx); Index: lib/MC/MCAsmStreamer.cpp =================================================================== --- lib/MC/MCAsmStreamer.cpp +++ lib/MC/MCAsmStreamer.cpp @@ -287,6 +287,9 @@ SMLoc Loc) override; void EmitWinEHHandlerData(SMLoc Loc) override; + void emitCGProfileEntry(const MCSymbol *From, const MCSymbol *To, + uint64_t Count) override; + void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI, bool PrintSchedInfo) override; @@ -1598,6 +1601,16 @@ EmitEOL(); } +void MCAsmStreamer::emitCGProfileEntry(const MCSymbol *From, const MCSymbol *To, + uint64_t Count) { + OS << "\t.cg_profile "; + From->print(OS, MAI); + OS << ", "; + To->print(OS, MAI); + OS << ", " << Count; + EmitEOL(); +} + void MCAsmStreamer::AddEncodingComment(const MCInst &Inst, const MCSubtargetInfo &STI, bool PrintSchedInfo) { Index: lib/MC/MCELFStreamer.cpp =================================================================== --- lib/MC/MCELFStreamer.cpp +++ lib/MC/MCELFStreamer.cpp @@ -365,6 +365,11 @@ ValueSize, MaxBytesToEmit); } +void MCELFStreamer::emitCGProfileEntry(const MCSymbol *From, const MCSymbol *To, + uint64_t Count) { + getAssembler().CGProfile.push_back({From, To, Count}); +} + void MCELFStreamer::EmitIdent(StringRef IdentString) { MCSection *Comment = getAssembler().getContext().getELFSection( ".comment", ELF::SHT_PROGBITS, ELF::SHF_MERGE | ELF::SHF_STRINGS, 1, ""); Index: lib/MC/MCParser/ELFAsmParser.cpp =================================================================== --- lib/MC/MCParser/ELFAsmParser.cpp +++ lib/MC/MCParser/ELFAsmParser.cpp @@ -85,6 +85,7 @@ addDirectiveHandler< &ELFAsmParser::ParseDirectiveSymbolAttribute>(".hidden"); addDirectiveHandler<&ELFAsmParser::ParseDirectiveSubsection>(".subsection"); + addDirectiveHandler<&ELFAsmParser::ParseDirectiveCGProfile>(".cg_profile"); } // FIXME: Part of this logic is duplicated in the MCELFStreamer. What is @@ -149,6 +150,7 @@ bool ParseDirectiveWeakref(StringRef, SMLoc); bool ParseDirectiveSymbolAttribute(StringRef, SMLoc); bool ParseDirectiveSubsection(StringRef, SMLoc); + bool ParseDirectiveCGProfile(StringRef, SMLoc); private: bool ParseSectionName(StringRef &SectionName); @@ -842,6 +844,40 @@ return false; } +/// ParseDirectiveCGProfile +/// ::= .cg_profile identifier, identifier, +bool ELFAsmParser::ParseDirectiveCGProfile(StringRef, SMLoc) { + StringRef From; + if (getParser().parseIdentifier(From)) + return TokError("expected identifier in directive"); + + if (getLexer().isNot(AsmToken::Comma)) + return TokError("expected a comma"); + Lex(); + + StringRef To; + if (getParser().parseIdentifier(To)) + return TokError("expected identifier in directive"); + + if (getLexer().isNot(AsmToken::Comma)) + return TokError("expected a comma"); + Lex(); + + int64_t Count; + if (getParser().parseIntToken( + Count, "expected integer count in '.cg_profile' directive")) + return true; + + if (getLexer().isNot(AsmToken::EndOfStatement)) + return TokError("unexpected token in directive"); + + MCSymbol *FromSym = getContext().getOrCreateSymbol(From); + MCSymbol *ToSym = getContext().getOrCreateSymbol(To); + + getStreamer().emitCGProfileEntry(FromSym, ToSym, Count); + return false; +} + namespace llvm { MCAsmParserExtension *createELFAsmParser() { Index: lib/MC/MCStreamer.cpp =================================================================== --- lib/MC/MCStreamer.cpp +++ lib/MC/MCStreamer.cpp @@ -652,6 +652,10 @@ getContext().reportError(Loc, "Chained unwind areas can't have handlers!"); } +void MCStreamer::emitCGProfileEntry(const MCSymbol *From, const MCSymbol *To, + uint64_t Count) { +} + static MCSection *getWinCFISection(MCContext &Context, unsigned *NextWinCFIID, MCSection *MainCFISec, const MCSection *TextSec) { Index: lib/Transforms/IPO/PassManagerBuilder.cpp =================================================================== --- lib/Transforms/IPO/PassManagerBuilder.cpp +++ lib/Transforms/IPO/PassManagerBuilder.cpp @@ -674,6 +674,8 @@ MPM.add(createConstantMergePass()); // Merge dup global constants } + MPM.add(createCGProfilePass()); + if (MergeFunctions) MPM.add(createMergeFunctionsPass()); Index: lib/Transforms/Instrumentation/CGProfile.cpp =================================================================== --- /dev/null +++ lib/Transforms/Instrumentation/CGProfile.cpp @@ -0,0 +1,104 @@ +//===-- CGProfile.cpp -----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Instrumentation.h" + +#include + +using namespace llvm; + +class CGProfilePass : public ModulePass { +public: + static char ID; + + CGProfilePass() : ModulePass(ID) { + initializeCGProfilePassPass( + *PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { return "CGProfilePass"; } + +private: + bool runOnModule(Module &M) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + } +}; + +bool CGProfilePass::runOnModule(Module &M) { + if (skipModule(M)) + return false; + + llvm::DenseMap, uint64_t> Counts; + + for (auto &F : M) { + if (F.isDeclaration()) + continue; + getAnalysis(F).getBPI(); + auto &BFI = getAnalysis(F).getBFI(); + for (auto &BB : F) { + Optional BBCount = BFI.getBlockProfileCount(&BB); + if (!BBCount) + continue; + for (auto &I : BB) { + auto *CI = dyn_cast(&I); + if (!CI) + continue; + Function *CalledF = CI->getCalledFunction(); + if (!CalledF || CalledF->isIntrinsic()) + continue; + + uint64_t &Count = + Counts[std::make_pair(F.getName(), CalledF->getName())]; + Count = SaturatingAdd(Count, *BBCount); + } + } + } + + if (Counts.empty()) + return false; + + LLVMContext &Context = M.getContext(); + MDBuilder MDB(Context); + std::vector Nodes; + + for (auto E : Counts) { + SmallVector Vals; + Vals.push_back(MDB.createString(E.first.first)); + Vals.push_back(MDB.createString(E.first.second)); + Vals.push_back(MDB.createConstant( + ConstantInt::get(Type::getInt64Ty(Context), E.second))); + Nodes.push_back(MDNode::get(Context, Vals)); + } + + M.addModuleFlag(Module::Append, "CG Profile", MDNode::get(Context, Nodes)); + + return true; +} + +char CGProfilePass::ID = 0; +INITIALIZE_PASS_BEGIN(CGProfilePass, "cg-profile", + "Generate profile information from the call graph.", false, + false) +INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass) +INITIALIZE_PASS_END(CGProfilePass, "cg-profile", + "Generate profile information from the call graph.", false, false) + +ModulePass *llvm::createCGProfilePass() { + return new CGProfilePass(); +} Index: lib/Transforms/Instrumentation/CMakeLists.txt =================================================================== --- lib/Transforms/Instrumentation/CMakeLists.txt +++ lib/Transforms/Instrumentation/CMakeLists.txt @@ -1,6 +1,7 @@ add_llvm_library(LLVMInstrumentation AddressSanitizer.cpp BoundsChecking.cpp + CGProfile.cpp DataFlowSanitizer.cpp GCOVProfiling.cpp MemorySanitizer.cpp Index: lib/Transforms/Instrumentation/Instrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/Instrumentation.cpp +++ lib/Transforms/Instrumentation/Instrumentation.cpp @@ -60,6 +60,7 @@ initializeAddressSanitizerModulePass(Registry); initializeBoundsCheckingLegacyPassPass(Registry); initializeGCOVProfilerLegacyPassPass(Registry); + initializeCGProfilePassPass(Registry); initializePGOInstrumentationGenLegacyPassPass(Registry); initializePGOInstrumentationUseLegacyPassPass(Registry); initializePGOIndirectCallPromotionLegacyPassPass(Registry); Index: tools/llvm-readobj/ELFDumper.cpp =================================================================== --- tools/llvm-readobj/ELFDumper.cpp +++ tools/llvm-readobj/ELFDumper.cpp @@ -98,6 +98,7 @@ using Elf_Vernaux = typename ELFT::Vernaux; \ using Elf_Verdef = typename ELFT::Verdef; \ using Elf_Verdaux = typename ELFT::Verdaux; \ + using Elf_CGProfile = typename ELFT::CGProfile; \ using uintX_t = typename ELFT::uint; namespace { @@ -162,6 +163,8 @@ void printHashHistogram() override; + void printCGProfile() override; + void printNotes() override; private: @@ -206,6 +209,7 @@ const Elf_Hash *HashTable = nullptr; const Elf_GnuHash *GnuHashTable = nullptr; const Elf_Shdr *DotSymtabSec = nullptr; + const Elf_Shdr *DotCGProfileSec = nullptr; StringRef DynSymtabName; ArrayRef ShndxTable; @@ -253,9 +257,11 @@ void getSectionNameIndex(const Elf_Sym *Symbol, const Elf_Sym *FirstSym, StringRef &SectionName, unsigned &SectionIndex) const; + StringRef getStaticSymbolName(uint32_t Index) const; void printSymbolsHelper(bool IsDynamic) const; const Elf_Shdr *getDotSymtabSec() const { return DotSymtabSec; } + const Elf_Shdr *getDotCGProfileSec() const { return DotCGProfileSec; } ArrayRef getShndxTable() const { return ShndxTable; } StringRef getDynamicStringTable() const { return DynamicStringTable; } const DynRegionInfo &getDynRelRegion() const { return DynRelRegion; } @@ -315,6 +321,7 @@ bool IsDynamic) = 0; virtual void printProgramHeaders(const ELFFile *Obj) = 0; virtual void printHashHistogram(const ELFFile *Obj) = 0; + virtual void printCGProfile(const ELFFile *Obj) = 0; virtual void printNotes(const ELFFile *Obj) = 0; virtual void printMipsGOT(const MipsGOTParser &Parser) = 0; virtual void printMipsPLT(const MipsGOTParser &Parser) = 0; @@ -344,6 +351,7 @@ size_t Offset) override; void printProgramHeaders(const ELFO *Obj) override; void printHashHistogram(const ELFFile *Obj) override; + void printCGProfile(const ELFFile *Obj) override; void printNotes(const ELFFile *Obj) override; void printMipsGOT(const MipsGOTParser &Parser) override; void printMipsPLT(const MipsGOTParser &Parser) override; @@ -404,6 +412,7 @@ void printDynamicRelocations(const ELFO *Obj) override; void printProgramHeaders(const ELFO *Obj) override; void printHashHistogram(const ELFFile *Obj) override; + void printCGProfile(const ELFFile *Obj) override; void printNotes(const ELFFile *Obj) override; void printMipsGOT(const MipsGOTParser &Parser) override; void printMipsPLT(const MipsGOTParser &Parser) override; @@ -730,6 +739,16 @@ } template +StringRef ELFDumper::getStaticSymbolName(uint32_t Index) const { + StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*DotSymtabSec)); + Elf_Sym_Range Syms = unwrapOrError(Obj->symbols(DotSymtabSec)); + if (Index >= Syms.size()) + reportError("Invalid symbol index"); + const Elf_Sym *Sym = &Syms[Index]; + return unwrapOrError(Sym->getName(StrTable)); +} + +template std::string ELFDumper::getFullSymbolName(const Elf_Sym *Symbol, StringRef StrTable, bool IsDynamic) const { @@ -1353,6 +1372,12 @@ reportError("Multiple SHT_GNU_verneed"); dot_gnu_version_r_sec = &Sec; break; + case ELF::SHT_NOTE: + if (unwrapOrError(Obj->getSectionName(&Sec)) != ".note.llvm.cgprofile") + break; + if (DotCGProfileSec != nullptr) + reportError("Multiple .note.llvm.cgprofile"); + DotCGProfileSec = &Sec; } } @@ -1497,6 +1522,10 @@ ELFDumperStyle->printHashHistogram(Obj); } +template void ELFDumper::printCGProfile() { + ELFDumperStyle->printCGProfile(Obj); +} + template void ELFDumper::printNotes() { ELFDumperStyle->printNotes(Obj); } @@ -3347,6 +3376,11 @@ } } +template +void GNUStyle::printCGProfile(const ELFFile *Obj) { + OS << "GNUStyle::printCGProfile not implemented\n"; +} + static std::string getGNUNoteTypeName(const uint32_t NT) { static const struct { uint32_t ID; @@ -4058,6 +4092,24 @@ } template +void LLVMStyle::printCGProfile(const ELFFile *Obj) { + ListScope L(W, "CGProfile"); + if (!this->dumper()->getDotCGProfileSec()) + return; + auto CGProfile = + unwrapOrError(Obj->template getSectionContentsAsArray( + this->dumper()->getDotCGProfileSec())); + for (const Elf_CGProfile &CGPE : CGProfile) { + DictScope D(W, "CGProfileEntry"); + W.printNumber("From", this->dumper()->getStaticSymbolName(CGPE.cgp_from), + CGPE.cgp_from); + W.printNumber("To", this->dumper()->getStaticSymbolName(CGPE.cgp_to), + CGPE.cgp_to); + W.printNumber("Weight", CGPE.cgp_weight); + } +} + +template void LLVMStyle::printNotes(const ELFFile *Obj) { W.startLine() << "printNotes not implemented!\n"; } Index: tools/llvm-readobj/ObjDumper.h =================================================================== --- tools/llvm-readobj/ObjDumper.h +++ tools/llvm-readobj/ObjDumper.h @@ -47,6 +47,7 @@ virtual void printVersionInfo() {} virtual void printGroupSections() {} virtual void printHashHistogram() {} + virtual void printCGProfile() {} virtual void printNotes() {} // Only implemented for ARM ELF at this time. Index: tools/llvm-readobj/llvm-readobj.cpp =================================================================== --- tools/llvm-readobj/llvm-readobj.cpp +++ tools/llvm-readobj/llvm-readobj.cpp @@ -281,6 +281,8 @@ cl::alias HashHistogramShort("I", cl::desc("Alias for -elf-hash-histogram"), cl::aliasopt(HashHistogram)); + cl::opt CGProfile("elf-cg-profile", cl::desc("Display callgraph profile section")); + cl::opt Output("elf-output-style", cl::desc("Specify ELF dump style"), cl::values(clEnumVal(LLVM, "LLVM default style"), @@ -436,6 +438,8 @@ Dumper->printGroupSections(); if (opts::HashHistogram) Dumper->printHashHistogram(); + if (opts::CGProfile) + Dumper->printCGProfile(); if (opts::Notes) Dumper->printNotes(); }