diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h b/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h --- a/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h @@ -10,9 +10,12 @@ #define LLVM_CODEGEN_BASICBLOCKSECTIONUTILS_H #include "llvm/ADT/STLExtras.h" +#include "llvm/Support/CommandLine.h" namespace llvm { +extern cl::opt BBSectionsColdTextPrefix; + class MachineFunction; class MachineBasicBlock; diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -79,6 +79,15 @@ using llvm::StringRef; using namespace llvm; +// Placing the cold clusters in a separate section mitigates against poor +// profiles and allows optimizations such as hugepage mapping to be applied at a +// section granularity. Where necessary, users should set this to ".text.split." +// which is recognized by lld via the `-z keep-text-section-prefix` flag. +cl::opt llvm::BBSectionsColdTextPrefix( + "bbsections-cold-text-prefix", + cl::desc("The text prefix to use for cold basic block clusters"), + cl::init(".text.unlikely."), cl::Hidden); + namespace { // This struct represents the cluster information for a machine basic block. diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -21,6 +21,7 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/CodeGen/BasicBlockSectionUtils.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" @@ -833,7 +834,7 @@ // name, or a unique ID for the section. SmallString<128> Name; if (MBB.getSectionID() == MBBSectionID::ColdSectionID) { - Name += ".text.unlikely."; + Name += BBSectionsColdTextPrefix; Name += MBB.getParent()->getName(); } else if (MBB.getSectionID() == MBBSectionID::ExceptionSectionID) { Name += ".text.eh."; diff --git a/llvm/test/CodeGen/X86/basic-block-sections-cold.ll b/llvm/test/CodeGen/X86/basic-block-sections-cold.ll --- a/llvm/test/CodeGen/X86/basic-block-sections-cold.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections-cold.ll @@ -2,25 +2,21 @@ ; Basic block with id 1 and 2 must be in the cold section. ; RUN: echo '!_Z3bazb' > %t ; RUN: echo '!!0' >> %t -; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=%t -unique-basic-block-section-names -bbsections-cold-text-prefix=".text.split." | FileCheck %s -check-prefix=LINUX-SPLIT -define void @_Z3bazb(i1 zeroext) nounwind { - %2 = alloca i8, align 1 - %3 = zext i1 %0 to i8 - store i8 %3, i8* %2, align 1 - %4 = load i8, i8* %2, align 1 - %5 = trunc i8 %4 to i1 - br i1 %5, label %6, label %8 +define void @_Z3bazb(i1 zeroext %0) nounwind { + br i1 %0, label %2, label %4 -6: ; preds = %1 - %7 = call i32 @_Z3barv() - br label %10 +2: ; preds = %1 + %3 = call i32 @_Z3barv() + br label %6 -8: ; preds = %1 - %9 = call i32 @_Z3foov() - br label %10 +4: ; preds = %1 + %5 = call i32 @_Z3foov() + br label %6 -10: ; preds = %8, %6 +6: ; preds = %2, %4 ret void } @@ -38,3 +34,9 @@ ; LINUX-SECTIONS-NOT: .section .text._Z3bazb._Z3bazb.2,"ax",@progbits,unique ; LINUX-SECTIONS: .LBB0_2: ; LINUX-SECTIONS: .size _Z3bazb, .Lfunc_end{{[0-9]}}-_Z3bazb + +; LINUX-SPLIT: .section .text.split._Z3bazb,"ax",@progbits +; LINUX-SPLIT-NEXT: _Z3bazb.cold: +; LINUX-SPLIT-NEXT: callq _Z3barv +; LINUX-SPLIT: .LBB0_2: +; LINUX-SPLIT: .LBB_END0_2: