Index: include/llvm/IR/Function.h =================================================================== --- include/llvm/IR/Function.h +++ include/llvm/IR/Function.h @@ -203,6 +203,12 @@ /// pgo data. Optional getEntryCount() const; + /// Set the section prefix for this function. + void setSectionPrefix(StringRef Prefix); + + /// Get the section prefix for this function. + Optional getSectionPrefix() const; + /// @brief Return true if the function has the attribute. bool hasFnAttribute(Attribute::AttrKind Kind) const { return AttributeSets.hasFnAttribute(Kind); Index: include/llvm/IR/LLVMContext.h =================================================================== --- include/llvm/IR/LLVMContext.h +++ include/llvm/IR/LLVMContext.h @@ -72,6 +72,7 @@ MD_align = 17, // "align" MD_loop = 18, // "llvm.loop" MD_type = 19, // "type" + MD_section_prefix = 20, // "section_prefix" }; /// Known operand bundle tag IDs, which always have the same value. All Index: include/llvm/IR/MDBuilder.h =================================================================== --- include/llvm/IR/MDBuilder.h +++ include/llvm/IR/MDBuilder.h @@ -66,6 +66,9 @@ /// Return metadata containing the entry count for a function. MDNode *createFunctionEntryCount(uint64_t Count); + /// Return metadata containing the section prefix for a function. + MDNode *createFunctionSectionPrefix(StringRef Prefix); + //===------------------------------------------------------------------===// // Range metadata. //===------------------------------------------------------------------===// Index: lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- lib/CodeGen/CodeGenPrepare.cpp +++ lib/CodeGen/CodeGenPrepare.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" @@ -119,6 +120,10 @@ "disable-preheader-prot", cl::Hidden, cl::init(false), cl::desc("Disable protection against removing loop preheaders")); +static cl::opt ProfileGuidedSectionPrefix( + "profile-guided-section-prefix", cl::Hidden, cl::init(true), + cl::desc("Use profile info to add section prefix for hot/cold functions")); + namespace { typedef SmallPtrSet SetOfInstrs; typedef PointerIntPair TypeIsSExt; @@ -168,6 +173,7 @@ void getAnalysisUsage(AnalysisUsage &AU) const override { // FIXME: When we can selectively preserve passes, preserve the domtree. + AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addRequired(); @@ -205,8 +211,11 @@ } char CodeGenPrepare::ID = 0; -INITIALIZE_TM_PASS(CodeGenPrepare, "codegenprepare", - "Optimize for code generation", false, false) +INITIALIZE_TM_PASS_BEGIN(CodeGenPrepare, "codegenprepare", + "Optimize for code generation", false, false) +INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) +INITIALIZE_TM_PASS_END(CodeGenPrepare, "codegenprepare", + "Optimize for code generation", false, false) FunctionPass *llvm::createCodeGenPreparePass(const TargetMachine *TM) { return new CodeGenPrepare(TM); @@ -231,6 +240,15 @@ LI = &getAnalysis().getLoopInfo(); OptSize = F.optForSize(); + if (ProfileGuidedSectionPrefix) { + ProfileSummaryInfo *PSI = + getAnalysis().getPSI(); + if (PSI->isHotFunction(&F)) + F.setSectionPrefix(".hot"); + else if (PSI->isColdFunction(&F)) + F.setSectionPrefix(".cold"); + } + /// This optimization identifies DIV instructions that can be /// profitably bypassed and carried out with a shorter, faster divide. if (!OptSize && TLI && TLI->isSlowDivBypassed()) { Index: lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -296,8 +296,12 @@ } else { Name = getSectionPrefixForGlobal(Kind); } - // FIXME: Extend the section prefix to include hotness catagories such as .hot - // or .unlikely for functions. + + if (const Function *F = dyn_cast(GV)) { + const auto &OptionalPrefix = F->getSectionPrefix(); + if (OptionalPrefix) + Name += *Prefix; + } if (EmitUniqueSection && UniqueSectionNames) { Name.push_back('.'); Index: lib/IR/Function.cpp =================================================================== --- lib/IR/Function.cpp +++ lib/IR/Function.cpp @@ -1274,3 +1274,17 @@ } return None; } + +void Function::setSectionPrefix(StringRef Prefix) { + MDBuilder MDB(getContext()); + setMetadata(LLVMContext::MD_section_prefix, MDB.createFunctionSectionPrefix(Prefix)); +} + +Optional Function::getSectionPrefix() const { + MDNode *MD = getMetadata(LLVMContext::MD_section_prefix); + if (MD && MD->getOperand(0)) + if (MDString *MDS = dyn_cast(MD->getOperand(0))) + if (MDS->getString().equals("function_section_prefix")) + return dyn_cast(MD->getOperand(1))->getString(); + return None; +} Index: lib/IR/LLVMContext.cpp =================================================================== --- lib/IR/LLVMContext.cpp +++ lib/IR/LLVMContext.cpp @@ -138,6 +138,11 @@ assert(TypeID == MD_type && "type kind id drifted"); (void)TypeID; + unsigned SectionPrefixID = getMDKindID("section_prefix"); + assert(SectionPrefixID == MD_section_prefix && + "section_prefix kind id drifted"); + (void)SectionPrefixID; + auto *DeoptEntry = pImpl->getOrInsertBundleTag("deopt"); assert(DeoptEntry->second == LLVMContext::OB_deopt && "deopt operand bundle id drifted!"); Index: lib/IR/MDBuilder.cpp =================================================================== --- lib/IR/MDBuilder.cpp +++ lib/IR/MDBuilder.cpp @@ -63,6 +63,12 @@ createConstant(ConstantInt::get(Int64Ty, Count))}); } +MDNode *MDBuilder::createFunctionSectionPrefix(StringRef Prefix) { + return MDNode::get(Context, + {createString("function_section_prefix"), + createString(Prefix)}); +} + MDNode *MDBuilder::createRange(const APInt &Lo, const APInt &Hi) { assert(Lo.getBitWidth() == Hi.getBitWidth() && "Mismatched bitwidths!"); Index: test/Transforms/CodeGenPrepare/section.ll =================================================================== --- /dev/null +++ test/Transforms/CodeGenPrepare/section.ll @@ -0,0 +1,38 @@ +; RUN: opt < %s -codegenprepare -S | FileCheck --check-prefixes=CHECK-OPT %s +; RUN: llc < %s -mtriple=x86_64-pc-linux-gnu -o - | FileCheck --check-prefixes=CHECK-LLC %s + +; This tests that hot/cold functions get correct section prefix assigned + +; CHECK-OPT: hot_func{{.*}}!section_prefix ![[HOT_ID:[0-9]+]] +; CHECK-LLC: .section .text.hot +; CHECK-LLC-NEXT: .globl hot_func +define void @hot_func() !prof !15 { + ret void +} + +; CHECK-OPT: cold_func{{.*}}!section_prefix ![[COLD_ID:[0-9]+]] +; CHECK-LLC: .section .text.cold +; CHECK-LLC-NEXT: .globl cold_func +define void @cold_func() !prof !16 { + ret void +} + +; CHECK-OPT: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"} +; CHECK-OPT: ![[COLD_ID]] = !{!"function_section_prefix", !".cold"} +!llvm.module.flags = !{!1} +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 1000} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 1000} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} +!15 = !{!"function_entry_count", i64 1000} +!16 = !{!"function_entry_count", i64 1}