diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -131,9 +131,15 @@ // SampleFDO is used, if a function doesn't have sample, it could be very // cold or it could be a new function never being sampled. Those functions // will be kept in the ".text.unknown" section. + // ".text.split." holds symbols which are split out from functions in other + // input sections. For example, with -fsplit-machine-functions, placing the + // cold parts in .text.split instead of .text.unlikely mitigates against poor + // profile inaccuracy. Techniques such as hugepage remapping can make + // conservative decisions at the section granularity. Additionally we find + // small improvement in icache and tlb metrics due to improved locality. if (config->zKeepTextSectionPrefix) for (StringRef v : {".text.hot.", ".text.unknown.", ".text.unlikely.", - ".text.startup.", ".text.exit."}) + ".text.startup.", ".text.exit.", ".text.split."}) if (isSectionPrefix(v, s->name)) return v.drop_back(); diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -79,6 +79,11 @@ using llvm::StringRef; using namespace llvm; +cl::opt ColdSectionTextPrefix( + "bbsections-cold-prefix", + cl::desc("The text prefix to use for cold basic block clusters."), + cl::init(".text.unlikely."), cl::Hidden); + namespace { // This struct represents the cluster information for a machine basic block. diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -66,6 +66,8 @@ using namespace llvm; using namespace dwarf; +extern cl::opt ColdSectionTextPrefix; + static void GetObjCImageInfo(Module &M, unsigned &Version, unsigned &Flags, StringRef &Section) { SmallVector ModuleFlags; @@ -873,7 +875,7 @@ // name, or a unique ID for the section. SmallString<128> Name; if (MBB.getSectionID() == MBBSectionID::ColdSectionID) { - Name += ".text.unlikely."; + Name += ColdSectionTextPrefix; Name += MBB.getParent()->getName(); } else if (MBB.getSectionID() == MBBSectionID::ExceptionSectionID) { Name += ".text.eh."; diff --git a/llvm/test/CodeGen/X86/basic-block-sections-cold.ll b/llvm/test/CodeGen/X86/basic-block-sections-cold.ll --- a/llvm/test/CodeGen/X86/basic-block-sections-cold.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections-cold.ll @@ -2,25 +2,21 @@ ; Basic block with id 1 and 2 must be in the cold section. ; RUN: echo '!_Z3bazb' > %t ; RUN: echo '!!0' >> %t -; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t -unique-basic-block-section-names -bbsections-cold-prefix=".text.split." | FileCheck %s -check-prefix=LINUX-SPLIT -define void @_Z3bazb(i1 zeroext) nounwind { - %2 = alloca i8, align 1 - %3 = zext i1 %0 to i8 - store i8 %3, i8* %2, align 1 - %4 = load i8, i8* %2, align 1 - %5 = trunc i8 %4 to i1 - br i1 %5, label %6, label %8 +define void @_Z3bazb(i1 zeroext %0) nounwind { + br i1 %0, label %2, label %4 -6: ; preds = %1 - %7 = call i32 @_Z3barv() - br label %10 +2: ; preds = %1 + %3 = call i32 @_Z3barv() + br label %6 -8: ; preds = %1 - %9 = call i32 @_Z3foov() - br label %10 +4: ; preds = %1 + %5 = call i32 @_Z3foov() + br label %6 -10: ; preds = %8, %6 +6: ; preds = %2, %4 ret void } @@ -38,3 +34,6 @@ ; LINUX-SECTIONS-NOT: .section .text._Z3bazb._Z3bazb.2,"ax",@progbits,unique ; LINUX-SECTIONS: .LBB0_2: ; LINUX-SECTIONS: .size _Z3bazb, .Lfunc_end{{[0-9]}}-_Z3bazb + +; LINUX-SPLIT: .section .text.split._Z3bazb,"ax",@progbits +; LINUX-SPLIT: _Z3bazb.cold: